From 2079142bc9716d7b780fc1c930224001e825cb96 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 21 Jun 2019 11:51:38 +0900 Subject: [PATCH] Add dependency resolver Change-Id: I8fabdbb8a824593b9a0a7e53cbf146ce14219da4 --- inc/multi_assistant_service_plugin.h | 2 + inc/multi_wakeup_recognizer.h | 1 + .../org.tizen.multi-assistant-service.spec | 4 +- plugins/wakeup-manager/CMakeLists.txt | 16 +- .../dependency-default/CMakeLists.txt | 61 ++ .../inc/dependency_default.h | 63 ++ .../inc/dependency_default_audio.h | 31 + .../inc/dependency_default_button.h | 23 + .../src/dependency_default.cpp | 58 ++ .../src/dependency_default_audio.cpp | 354 +++++++++ .../src/dependency_default_button.cpp | 15 + .../wakeup-manager/inc/dependency_resolver.h | 81 ++ plugins/wakeup-manager/inc/heap_tracer.h | 81 ++ .../wakeup-manager/inc/wakeup_audio_manager.h | 44 +- .../wakeup-manager/inc/wakeup_interfaces.h | 1 + plugins/wakeup-manager/inc/wakeup_manager.h | 9 + .../inc/wakeup_manager_wrapper.h | 34 +- .../src/dependency_resolver.cpp | 224 ++++++ plugins/wakeup-manager/src/heap_tracer.cpp | 227 ++++++ .../src/wakeup_audio_manager.cpp | 712 ++---------------- .../src/wakeup_engine_manager.cpp | 1 + plugins/wakeup-manager/src/wakeup_manager.cpp | 199 ++--- .../src/wakeup_manager_wrapper.cpp | 31 +- src/multi_assistant_service.c | 2 + src/multi_assistant_service_plugin.c | 2 +- 25 files changed, 1415 insertions(+), 861 deletions(-) create mode 100644 plugins/wakeup-manager/dependency-default/CMakeLists.txt create mode 100644 plugins/wakeup-manager/dependency-default/inc/dependency_default.h create mode 100644 plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h create mode 100644 plugins/wakeup-manager/dependency-default/inc/dependency_default_button.h create mode 100644 plugins/wakeup-manager/dependency-default/src/dependency_default.cpp create mode 100644 plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp create mode 100644 plugins/wakeup-manager/dependency-default/src/dependency_default_button.cpp create mode 100644 plugins/wakeup-manager/inc/dependency_resolver.h create mode 100644 plugins/wakeup-manager/inc/heap_tracer.h create mode 100644 plugins/wakeup-manager/src/dependency_resolver.cpp create mode 100644 plugins/wakeup-manager/src/heap_tracer.cpp diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 325b988..06f58b5 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -30,6 +30,8 @@ extern "C" { typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH = MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP, } ma_plugin_event_e; typedef struct { diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index df8df1a..394ae49 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -30,6 +30,7 @@ typedef struct { const char *wakeup_word; const char *wakeup_language; const char *wakeup_voice_id; + const char *wakeup_engine; float wakeup_confidence_score; long wakeup_start_time; diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 602abe7..ca96f19 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -61,9 +61,6 @@ LDFLAGS="$LDFLAGS -Wl,-z -Wl,nodelete" export LDFLAGS cmake \ -%if "%{tizen_profile_name}" == "tv" - -D_TV_PRODUCT=TRUE \ -%endif -DCMAKE_INSTALL_PREFIX=%{_appdir} \ -DTZ_SYS_RO_SHARE=%TZ_SYS_RO_SHARE \ @@ -101,6 +98,7 @@ exit 0 /usr/share/license/* /usr/share/packages/org.tizen.multi-assistant-service.xml %{TZ_SYS_RO_SHARE}/multiassistant/libma-wakeup-manager.so +%{TZ_SYS_RO_SHARE}/multiassistant/libma-dependency-default.so %{_appdir}/author-signature.xml %{_appdir}/signature1.xml #%{_appdir}/lib/* diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 55fccc8..9bb3141 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -25,15 +25,6 @@ SET(WMPKG_CHECK_MODULES pkgmgr-info ) -IF("${_TV_PRODUCT}" STREQUAL "TRUE") - SET(WMPKG_CHECK_MODULES ${WMPKG_CHECK_MODULES} - capi-network-bluetooth - capi-network-bluetooth-tv - vd-win-util - farfield-voice-api - ) -ENDIF() - INCLUDE(FindPkgConfig) pkg_check_modules(wmpkgs REQUIRED ${WMPKG_CHECK_MODULES}) MESSAGE("Modules : ${WMPKG_CHECK_MODULES}") @@ -54,6 +45,8 @@ SET(SRCS src/wakeup_audio_manager.cpp src/wakeup_engine_manager.cpp src/wakeup_manager_wrapper.cpp + src/dependency_resolver.cpp + src/heap_tracer.cpp ) FOREACH(flag ${wmpkgs_CFLAGS}) @@ -63,9 +56,6 @@ ENDFOREACH(flag) SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) -IF("${_TV_PRODUCT}" STREQUAL "TRUE") - SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -DTV_PRODUCT") -ENDIF() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") @@ -75,3 +65,5 @@ ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${wmpkgs_LDFLAGS} -ldl ${EXTRA_LDFLAGS}) MESSAGE("LDFLAG : ${wmpkgs_LDFLAGS}") INSTALL(FILES ${CMAKE_SOURCE_DIR}/plugins/wakeup-manager/libma-wakeup-manager.so DESTINATION ${TZ_SYS_RO_SHARE}/multiassistant/) + +ADD_SUBDIRECTORY(dependency-default) diff --git a/plugins/wakeup-manager/dependency-default/CMakeLists.txt b/plugins/wakeup-manager/dependency-default/CMakeLists.txt new file mode 100644 index 0000000..2fbe237 --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/CMakeLists.txt @@ -0,0 +1,61 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +PROJECT(ma-dependency-default) + +SET(PREFIX ${CMAKE_INSTALL_PREFIX}) +SET(PACKAGE ${PROJECT_NAME}) + +SET(CMAKE_SKIP_BUILD_RPATH TRUE) + +INCLUDE_DIRECTORIES( + ${CMAKE_SOURCE_DIR}/inc + ./inc + ) + +SET(DDPKG_CHECK_MODULES + appcore-agent + ecore + ecore-wl2 + dlog + capi-appfw-app-manager + capi-network-connection + capi-media-audio-io + capi-media-sound-manager + eina + vconf + pkgmgr-info +) + +INCLUDE(FindPkgConfig) +pkg_check_modules(ddpkgs REQUIRED ${DDPKG_CHECK_MODULES}) +MESSAGE("Modules : ${DDPKG_CHECK_MODULES}") + +IF("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE "Release") +ENDIF("${CMAKE_BUILD_TYPE}" STREQUAL "") +MESSAGE("Build type: ${CMAKE_BUILD_TYPE}") + +ADD_DEFINITIONS("-DPREFIX=\"${CMAKE_INSTALL_PREFIX}\"") +ADD_DEFINITIONS("-DPACKAGE=\"${PACKAGE}\"") + +SET(SRCS + src/dependency_default.cpp + src/dependency_default_audio.cpp + src/dependency_default_button.cpp +) + +FOREACH(flag ${ddpkgs_CFLAGS}) + SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} ${flag}") +ENDFOREACH(flag) + + + +SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") + +ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) + +# Install libraries +TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${ddpkgs_LDFLAGS} -ldl ${EXTRA_LDFLAGS}) +MESSAGE("LDFLAG : ${ddpkgs_LDFLAGS}") +INSTALL(FILES ${CMAKE_SOURCE_DIR}/plugins/wakeup-manager/dependency-default/lib${PACKAGE}.so DESTINATION ${TZ_SYS_RO_SHARE}/multiassistant/) diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h new file mode 100644 index 0000000..51fd6d2 --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#ifndef _DEPENDENCY_DEFAULT_H_ +#define _DEPENDENCY_DEFAULT_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + WAKEUP_SPEECH_STREAMING_EVENT_FAIL = -1, /**< Failed */ + WAKEUP_SPEECH_STREAMING_EVENT_START = 1, /**< Start event */ + WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE = 2, /**< Continue event */ + WAKEUP_SPEECH_STREAMING_EVENT_FINISH = 3 /**< Finish event */ +} wakeup_speech_streaming_event_e; + +typedef enum { + MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH = MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP, +} ma_plugin_event_e; + +typedef void (*mas_dependency_error_cb)(int error, const char* err_msg, void* user_data); + +typedef int (*mas_proxy_process_event)(int event, void* data, int len); +typedef int (*mas_proxy_feed_audio_data)(wakeup_speech_streaming_event_e event, void* buffer, int len); + +typedef struct { + mas_proxy_process_event process_event; + mas_proxy_feed_audio_data feed_audio_data; +} mas_proxy_interface; + +EXPORT_API int mas_dependency_initialize(mas_proxy_interface interfaces, int *dependency_version); +EXPORT_API int mas_dependency_deinitialize(void); +EXPORT_API int mas_dependency_set_error_callback(mas_dependency_error_cb callback, void* user_data); +EXPORT_API int mas_dependency_start_recording(void); +EXPORT_API int mas_dependency_stop_recording(void); +EXPORT_API int mas_dependency_change_system_volume(void); +EXPORT_API int mas_dependency_recover_system_volume(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _DEPENDENCY_DEFAULT_H_ */ diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h new file mode 100644 index 0000000..b46dcf8 --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _DEPENDENCY_DEFAULT_AUDIO_H_ +#define _DEPENDENCY_DEFAULT_AUDIO_H_ + +void dependency_default_audio_initialize(mas_proxy_interface interface); +void dependency_default_audio_deinitialize(); + +void dependency_default_audio_start_recording(); +void dependency_default_audio_stop_recording(); + +void dependency_default_audio_voice_key_pressed_set(bool pressed); + +void dependency_default_audio_change_system_volume(); +void dependency_default_audio_recover_system_volume(); + +#endif //_DEPENDENCY_DEFAULT_AUDIO_H_ diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default_button.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default_button.h new file mode 100644 index 0000000..4639c4b --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default_button.h @@ -0,0 +1,23 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _DEPENDENCY_DEFAULT_BUTTON_H_ +#define _DEPENDENCY_DEFAULT_BUTTON_H_ + +void dependency_default_button_initialize(mas_proxy_interface interface); +void dependency_default_button_deinitialize(); + +#endif //_DEPENDENCY_DEFAULT_BUTTON_H_ diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp new file mode 100644 index 0000000..c14145d --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp @@ -0,0 +1,58 @@ +#include "dependency_default.h" +#include "dependency_default_audio.h" +#include "dependency_default_button.h" + +static mas_proxy_interface g_proxy_interface; +const int g_dependency_version = 1; + +int mas_dependency_initialize(mas_proxy_interface interface, int *dependency_version) +{ + g_proxy_interface = interface; + + dependency_default_audio_initialize(interface); + dependency_default_button_initialize(interface); + + if (dependency_version) { + *dependency_version = g_dependency_version; + } + + return 0; +} + +int mas_dependency_deinitialize(void) +{ + dependency_default_audio_deinitialize(); + dependency_default_button_deinitialize(); + + return 0; +} + +int mas_dependency_set_error_callback(mas_dependency_error_cb callback, void* user_data) +{ + return 0; +} + +int mas_dependency_start_recording(void) +{ + dependency_default_audio_start_recording(); + return 0; +} + +int mas_dependency_stop_recording(void) +{ + dependency_default_audio_stop_recording(); + g_proxy_interface.process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP, nullptr, 0); + return 0; +} + +int mas_dependency_change_system_volume(void) +{ + dependency_default_audio_change_system_volume(); + return 0; +} + +int mas_dependency_recover_system_volume() +{ + dependency_default_audio_recover_system_volume(); + return 0; +} \ No newline at end of file diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp new file mode 100644 index 0000000..4bceca6 --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp @@ -0,0 +1,354 @@ +#include "dependency_default.h" +#include "dependency_default_audio.h" + +static bool g_voice_key_pressed = false; +static mas_proxy_interface g_proxy_interface; + +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace std; + +static audio_in_h g_audio_in = NULL; + +static sound_stream_info_h g_volume_stream = NULL; +static virtual_sound_stream_h g_virtual_sound_stream = NULL; + +static int g_device_id = -1; +static sound_stream_info_h g_stream_info = NULL; + +static thread g_recorder_thread; +static atomic_bool g_stop_recorder_thread{false}; + +static long get_current_milliseconds_after_epoch() +{ + auto now = chrono::system_clock::now(); + auto now_ms = chrono::time_point_cast(now); + /* number of milliseconds since the epoch of system_clock */ + auto value = now_ms.time_since_epoch(); + + return value.count(); +} + +static int _apply_device_for_stream_routing(void *user_data) +{ + sound_device_list_h deviceList = NULL; + sound_device_h device = NULL; + sound_device_type_e type; + sound_device_io_direction_e ioDirection; + + if (0 != sound_manager_get_device_list(SOUND_DEVICE_IO_DIRECTION_IN_MASK, &deviceList)) { + LOGE("[Recorder ERROR] Fail to get current device list"); + return -1; + } + + bool isFound = false; + while (0 == sound_manager_get_next_device(deviceList, &device)) { + if (0 != sound_manager_get_device_type(device, &type)) { + LOGE("[Recorder ERROR] Fail to get device type"); + continue; + } + if (0 != sound_manager_get_device_io_direction(device, &ioDirection)) { + LOGE("[Recorder ERROR] Fail to get device io direction"); + continue; + } + + if (SOUND_DEVICE_USB_AUDIO == type && SOUND_DEVICE_IO_DIRECTION_IN == ioDirection) { + if (0 != sound_manager_add_device_for_stream_routing(g_stream_info, device)) { + LOGE("[Recorder ERROR] Fail to add device"); + continue; + } + if (0 != sound_manager_apply_stream_routing(g_stream_info)) { + LOGE("[Recorder ERROR] Fail to apply stream routing"); + continue; + } + isFound = true; + break; + } + } + + sound_manager_free_device_list(deviceList); + deviceList = NULL; + + if (true != isFound) { + LOGI("[Recorder] No USB device"); + } else { + LOGD("[Recorder] Apply device for stream routing"); + } + return 0; +} + +static void _device_connection_changed_cb(sound_device_h device, bool isConnected, void *user_data) +{ + sound_device_type_e type; + if (isConnected) { + if (0 != sound_manager_get_device_type(device, &type)) { + LOGE("[Recorder ERROR] Fail to get device type"); + return; + } + if (type == SOUND_DEVICE_USB_AUDIO) { + if (0 != sound_manager_remove_device_for_stream_routing(g_stream_info, device)) + LOGE("[Recorder ERROR] Fail to remove device"); + + if (0 != sound_manager_add_device_for_stream_routing(g_stream_info, device)) { + LOGE("[Recorder ERROR] Fail to add device"); + return; + } + if (0 != sound_manager_apply_stream_routing(g_stream_info)) { + LOGE("[Recorder ERROR} Fail to apply stream routing"); + return; + } + LOGD("[Recorder] Apply device for stream routing"); + } + } + return; +} + +void dependency_default_audio_initialize(mas_proxy_interface interfaces) +{ + const int rate = 16000; + const audio_channel_e channel = AUDIO_CHANNEL_MONO; + const audio_sample_type_e type = AUDIO_SAMPLE_TYPE_S16_LE; + + g_proxy_interface = interfaces; + + int ret = audio_in_create(rate, channel, type, &g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGE("[Recorder ERROR] Rate(%d) Channel(%d) Type(%d)", rate, channel, type); + LOGE("[Recorder ERROR] Fail to create audio handle : %d", ret); + return; + } + + ret = sound_manager_add_device_connection_changed_cb(SOUND_DEVICE_IO_DIRECTION_IN_MASK, _device_connection_changed_cb, NULL, &g_device_id); + if (0 != ret) { + LOGE("[Recorder ERROR] Fail to add device connection changed callback"); + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + + ret = sound_manager_create_stream_information_internal( + SOUND_STREAM_TYPE_VOICE_RECOGNITION_SERVICE, NULL, NULL, &g_stream_info); + if (0 != ret) { + LOGE("[Recorder ERROR] Fail to create stream info"); + sound_manager_remove_device_connection_changed_cb(g_device_id); + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + + ret = _apply_device_for_stream_routing(NULL); + if (0 != ret) { + LOGE("[Recorder ERROR] Fail to apply device for stream routing: %d", ret); + sound_manager_remove_device_connection_changed_cb(g_device_id); + sound_manager_destroy_stream_information(g_stream_info); + g_stream_info = NULL; + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + + ret = audio_in_set_sound_stream_info(g_audio_in, g_stream_info); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGE("[Recorder ERROR] Fail to set stream info : %d", ret); + sound_manager_remove_device_connection_changed_cb(g_device_id); + sound_manager_destroy_stream_information(g_stream_info); + g_stream_info = NULL; + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + + ret = audio_in_prepare(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + if (AUDIO_IO_ERROR_SOUND_POLICY == ret) + { + LOGE("[Recorder ERROR] Audio is busy."); + } else { + LOGE("[Recorder ERROR] Fail to start audio : %d", ret); + } + sound_manager_remove_device_connection_changed_cb(g_device_id); + sound_manager_destroy_stream_information(g_stream_info); + g_stream_info = NULL; + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + + ret = audio_in_pause(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGE("[Recorder ERROR] Fail to pause audio in : %d", ret); + } +} + +void dependency_default_audio_deinitialize() +{ + int ret = 0; + if (g_virtual_sound_stream) { + sound_manager_stop_virtual_stream(g_virtual_sound_stream); + ret = sound_manager_destroy_virtual_stream(g_virtual_sound_stream); + if (0 != ret) { + LOGE("[Audio ERROR] Fail to destroy virtual stream, ret(%d)", ret); + } + g_virtual_sound_stream = NULL; + } + + if (g_device_id) { + ret = sound_manager_remove_device_connection_changed_cb(g_device_id); + if (0 != ret) + LOGE("[Audio ERROR] Fail to remove device connection changed callback, ret(%d)", ret); + } + + if (g_volume_stream) { + ret = sound_manager_destroy_stream_information(g_volume_stream); + if (0 != ret) { + LOGE("[Audio ERROR] Fail to destroy stream information, ret(%d)", ret); + } + g_volume_stream = NULL; + } + + ret = audio_in_unprepare(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to stop audio : %d", ret); + } + + if (0 != sound_manager_destroy_stream_information(g_stream_info)) { + LOGD("[Recorder ERROR] Fail to destroy stream info"); + } + g_stream_info = NULL; + + ret = audio_in_destroy(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret); + } + g_audio_in = NULL; +} + +static void recorder_thread_func() +{ + const int FRAME_LENGTH = 160; + const int BUFFER_LENGTH = FRAME_LENGTH * 2; + + static int buffer_count = 0; + + while (!(g_stop_recorder_thread.load())) { + unsigned char buffer[BUFFER_LENGTH]; + memset(buffer, '\0', BUFFER_LENGTH); + + int read_bytes = audio_in_read(g_audio_in, buffer, BUFFER_LENGTH); + if (0 > read_bytes) { + LOGE("[Recorder WARNING] Fail to read audio : %d", read_bytes); + break; + } + + g_proxy_interface.feed_audio_data(WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE, + buffer, read_bytes); + + /* Audio read log */ + if (0 == buffer_count % 300) { + LOGD("[Recorder][%d] Recording... : read_size(%d)", buffer_count, read_bytes); + } + + buffer_count++; + } +} + +void dependency_default_audio_start_recording() +{ + int ret = audio_in_resume(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); + } + g_stop_recorder_thread.store(false); + LOGD("Starting recorder thread"); + g_recorder_thread = thread(recorder_thread_func); +} + +void dependency_default_audio_stop_recording() +{ + if (g_recorder_thread.joinable()) { + LOGD("g_recorder_thread is joinable, trying join()"); + g_stop_recorder_thread.store(true); + g_recorder_thread.join(); + } + int ret = audio_in_pause(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); + } +} + +void dependency_default_audio_voice_key_pressed_set(bool pressed) +{ + g_voice_key_pressed = pressed; +} + +void dependency_default_audio_change_system_volume() +{ + int ret; + + if (!g_volume_stream) { + ret = sound_manager_create_stream_information_internal(SOUND_STREAM_TYPE_VOICE_RECOGNITION_SERVICE, NULL, NULL, &g_volume_stream); + if (0 != ret) { + LOGD("[Audio] Fail to create stream information, ret(%d)", ret); + return; + } + } + + if (!g_virtual_sound_stream) { + ret = sound_manager_create_virtual_stream(g_volume_stream, &g_virtual_sound_stream); + if (0 != ret) { + if (g_volume_stream) { + sound_manager_destroy_stream_information(g_volume_stream); + g_volume_stream = NULL; + } + LOGD("[Audio ERROR] Fail to create virtual stream, ret(%d)", ret); + return; + } + } + if (g_virtual_sound_stream) { + ret = sound_manager_start_virtual_stream(g_virtual_sound_stream); + if (0 != ret) { + if (g_virtual_sound_stream) { + sound_manager_destroy_virtual_stream(g_virtual_sound_stream); + g_virtual_sound_stream = NULL; + } + if (g_volume_stream) { + sound_manager_destroy_stream_information(g_volume_stream); + g_volume_stream = NULL; + } + LOGD("[Audio ERROR] Fail to start virtual stream, ret(%d)", ret); + return; + } + } +} + +void dependency_default_audio_recover_system_volume() +{ + int ret; + if (g_virtual_sound_stream) { + ret = sound_manager_stop_virtual_stream(g_virtual_sound_stream); + if (0 != ret) { + LOGD("[Audio ERROR] Fail to stop virtual stream, ret(%d)", ret); + } + ret = sound_manager_destroy_virtual_stream(g_virtual_sound_stream); + if (0 != ret) { + LOGD("[Audio ERROR] Fail to destroy virtual stream, ret(%d)", ret); + } + g_virtual_sound_stream = NULL; + } + + if (g_volume_stream) { + ret = sound_manager_destroy_stream_information(g_volume_stream); + if (0 != ret) { + LOGD("[Audio ERROR] Fail to destroy stream information, ret(%d)", ret); + } + g_volume_stream = NULL; + } +} \ No newline at end of file diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_button.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_button.cpp new file mode 100644 index 0000000..5b39979 --- /dev/null +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_button.cpp @@ -0,0 +1,15 @@ +#include "dependency_default.h" +#include "dependency_default_button.h" + +#include + +static mas_proxy_interface g_proxy_interface; + +void dependency_default_button_initialize(mas_proxy_interface interface) +{ + g_proxy_interface = interface; +} + +void dependency_default_button_deinitialize() +{ +} diff --git a/plugins/wakeup-manager/inc/dependency_resolver.h b/plugins/wakeup-manager/inc/dependency_resolver.h new file mode 100644 index 0000000..6693ba5 --- /dev/null +++ b/plugins/wakeup-manager/inc/dependency_resolver.h @@ -0,0 +1,81 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#ifndef _DEPENDENCY_RESOLVER_H_ +#define _DEPENDENCY_RESOLVER_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*mas_dependency_error_cb)(int error, const char* err_msg, void* user_data); + +typedef int (*mas_proxy_process_event)(int event, void* data, int len); +typedef int (*mas_proxy_feed_audio_data)(wakeup_speech_streaming_event_e event, void* buffer, int len); + +typedef struct { + mas_proxy_process_event process_event; + mas_proxy_feed_audio_data feed_audio_data; +} mas_proxy_interface; + +int dependency_resolver_initialize(mas_proxy_interface interface); +int dependency_resolver_deinitialize(void); +int dependency_resolver_set_error_callback(mas_dependency_error_cb callback, void* user_data); +int dependency_resolver_start_recording(); +int dependency_resolver_stop_recording(); +int dependency_resolver_change_system_volume(); +int dependency_resolver_recover_system_volume(); + +/************************************************************************************** + *** Definitions for dependencies + *************************************************************************************/ +#define MAS_DEPENDENCY_DEFAULT_PATH tzplatform_mkpath(tzplatform_getid("TZ_SYS_RO_SHARE"), "multiassistant/") +#define MAS_DEPENDENCY_DEFAULT_FILENAME "libma-dependency-default.so" +#define MAS_DEPENDENCY_MODULE_PATH "db/multi-assistant/dependency_module_path" + +#define MAS_DEPENDENCY_FUNC_INITIALIZE "mas_dependency_initialize" +typedef int (*mas_dependency_initialize)(mas_proxy_interface interface, int *dependency_version); +#define MAS_DEPENDENCY_FUNC_DEINITIALIZE "mas_dependency_deinitialize" +typedef int (*mas_dependency_deinitialize)(void); +#define MAS_DEPENDENCY_FUNC_SET_ERROR_CALLBACK "mas_dependency_set_error_callback" +typedef int (*mas_dependency_set_error_callback)(mas_dependency_error_cb callback, void* user_data); +#define MAS_DEPENDENCY_FUNC_START_RECORDING "mas_dependency_start_recording" +typedef int (*mas_dependency_start_recording)(void); +#define MAS_DEPENDENCY_FUNC_STOP_RECORDING "mas_dependency_stop_recording" +typedef int (*mas_dependency_stop_recording)(void); +#define MAS_DEPENDENCY_FUNC_CHANGE_SYSTEM_VOLUME "mas_dependency_change_system_volume" +typedef int (*mas_dependency_change_system_volume)(void); +#define MAS_DEPENDENCY_FUNC_RECOVER_SYSTEM_VOLUME "mas_dependency_recover_system_volume" +typedef int (*mas_dependency_recover_system_volume)(void); + +typedef struct { + mas_dependency_initialize initialize; + mas_dependency_deinitialize deinitialize; + mas_dependency_set_error_callback set_error_callback; + mas_dependency_start_recording start_recording; + mas_dependency_stop_recording stop_recording; + mas_dependency_change_system_volume change_system_volume; + mas_dependency_recover_system_volume recover_system_volume; +} mas_dependency; + +#ifdef __cplusplus +} +#endif + +#endif /* _DEPENDENCY_RESOLVER_H_ */ diff --git a/plugins/wakeup-manager/inc/heap_tracer.h b/plugins/wakeup-manager/inc/heap_tracer.h new file mode 100644 index 0000000..fc7df29 --- /dev/null +++ b/plugins/wakeup-manager/inc/heap_tracer.h @@ -0,0 +1,81 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __HEAP_TRACER_H__ +#define __HEAP_TRACER_H__ + +#include +#include +#include +#include +#include + +#ifndef __MODULE__ +#define __MODULE__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) +#endif + +#define vm_calloc_simple(nmemb, size) vm_calloc(nmemb, size, "%s: %s(%d)", __MODULE__, __func__, __LINE__) +#define vm_malloc_simple(size) vm_malloc(size, "%s: %s(%d)", __MODULE__, __func__, __LINE__) +#define vm_free_simple(ptr) vm_free(ptr, "%s: %s(%d)", __MODULE__, __func__, __LINE__) +#define vm_strdup_simple(ptr) vm_strdup(ptr, "%s: %s(%d)", __MODULE__, __func__, __LINE__) +#define vm_mark_unmanaged_simple(ptr) vm_mark_unmanaged(ptr, "%s: %s(%d)", __MODULE__, __func__, __LINE__) + +void *vm_calloc(size_t nmemb, size_t size, const char *fmt, ...); +void *vm_malloc(size_t size, const char *fmt, ...); +void vm_free(void *ptr, const char *fmt, ...); +char *vm_strdup(const char *s, const char *fmt, ...); +void *vm_mark_unmanaged(void *ptr, const char *fmt, ...); + +class HeapTracer { +public: + HeapTracer(); + virtual ~HeapTracer(); + + bool Insert(void *ptr, std::string description, size_t size = 0); + bool Delete(void *ptr, std::string description); + std::string Find(void *ptr); + + void MarkUnmanaged(void *ptr, std::string description); + + bool Empty(); + void Trace(); +protected: + typedef struct { + void *ptr{nullptr}; + std::string description; + int size{0}; + } AllocationEntry; + + std::mutex mManagedTableMutex; + std::unordered_map mManagedTable; + + std::mutex mUnmanagedRecordsMutex; + std::list mUnmanagedRecords; + size_t mUnmanagedRecordsSize{0}; + + enum class EntryOperation { + INSERT, + DELETE, + }; + typedef struct { + AllocationEntry entry; + EntryOperation operation; + struct timespec ts; + } EntryHistory; + std::list mEntryHistory; +}; + +#endif /* __HEAP_TRACER_H__ */ \ No newline at end of file diff --git a/plugins/wakeup-manager/inc/wakeup_audio_manager.h b/plugins/wakeup-manager/inc/wakeup_audio_manager.h index de42d3c..0e1bc6b 100644 --- a/plugins/wakeup-manager/inc/wakeup_audio_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_audio_manager.h @@ -28,7 +28,6 @@ #include #include -#include namespace multiassistant { @@ -65,15 +64,10 @@ public: void start_recording(); void stop_recording(); - void add_speech_data(wakeup_speech_data& data); - void clear_speech_data(); - void finalize_speech_data(); + void feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len); + void finalize_audio_data(); - void add_background_data(wakeup_speech_data& data, long time); - - void notify_audio_data_recording(long time, void* data, int len); - - void start_streaming_current_utterance_data(bool from_start_time = false, long start_time = 0); + void start_streaming_current_utterance_data(long start_time = 0); void stop_streaming_current_utterance_data(); void start_streaming_previous_utterance_data(); @@ -82,44 +76,32 @@ public: void start_streaming_follow_up_data(); void stop_streaming_follow_up_data(); - void voice_key_pressed_set(bool pressed); - bool voice_key_pressed_get(); - void change_system_volume(); void recover_system_volume(); - int mDeviceId{-1}; - sound_stream_info_h mStreamInfo{nullptr}; - private: - void recorder_thread_func(void); - void streaming_speech_data_thread_func(); - void streaming_previous_speech_data_thread_func(); - void streaming_background_data_thread_func(long start_time); - - vector mObservers; - - audio_in_h mAudioIn{nullptr}; + void add_audio_data(wakeup_speech_data& data, long time); + void notify_audio_data_recording(long time, void* data, int len); - sound_stream_info_h mVolumeStream{nullptr}; - virtual_sound_stream_h mVirtualSoundStream{nullptr}; + void streaming_previous_audio_data_thread_func(); + void streaming_audio_data_thread_func(long start_time); - thread mRecorderThread; - atomic_bool mStopRecorderThread{false}; + vector mObservers; thread mStreamingThread; atomic_bool mStopStreamingThread{false}; thread mStreamingPreviousThread; atomic_bool mStopStreamingPreviousThread{false}; - static constexpr long mBackgroundRecordingDurationMilliseconds = 10 * 1000; + + static constexpr long mAudioRecordingDurationMilliseconds = 10 * 1000; typedef struct { long time; wakeup_speech_data data; } wakeup_speech_data_with_time; - vector mSpeechData; - vector mPreviousSpeechData; - list mBackgroundData; + vector mPreviousAudioData; + list mAudioData; + mutex mMutex; bool mVoiceKeyPressed{false}; }; diff --git a/plugins/wakeup-manager/inc/wakeup_interfaces.h b/plugins/wakeup-manager/inc/wakeup_interfaces.h index 7303ca6..6d12de9 100644 --- a/plugins/wakeup-manager/inc/wakeup_interfaces.h +++ b/plugins/wakeup-manager/inc/wakeup_interfaces.h @@ -22,6 +22,7 @@ typedef struct { const char *wakeup_word; const char *wakeup_language; const char *wakeup_voice_id; + const char *wakeup_engine; float wakeup_confidence_score; long wakeup_start_time; diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index 2a7d80e..c76e3a0 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -48,6 +48,10 @@ public: wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; }; +/* If a wakeup event is raised by pressing a voice key, + the wakeup_engine attribute of wakeup_event_info will have the following value */ +#define WAKEUP_ENGINE_VOICE_KEY "voice_key" + class CWakeupManager { public: @@ -100,6 +104,8 @@ public: bool change_manager_state(wakeup_manager_state_e state); wakeup_manager_state_e get_manager_state(); + + void feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len); private: class CEngineEventObserver : public IEngineEventObserver { @@ -144,7 +150,9 @@ private: string appid; vector languageList; } LanguageInfo; + void initialize_wakeup_policy(); + vector mLanguageInfo; vector mObservers; @@ -168,6 +176,7 @@ private: Ecore_Timer* mStreamingDurationTimer{nullptr}; wakeup_manager_state_e mWakeupManagerState{WAKEUP_MANAGER_STATE_INACTIVE}; + wakeup_event_info mLastWakeupEventInfo; }; diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index 6984f06..e3ba274 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -20,6 +20,7 @@ #include #include +#include #include #include "wakeup_interfaces.h" @@ -28,33 +29,6 @@ extern "C" { #endif -#ifndef EXPORT_API - #if defined _WIN32 || defined __CYGWIN__ - #ifdef BUILDING_DLL - #ifdef __GNUC__ - #define EXPORT_API __attribute__ ((dllexport)) - #else - #define EXPORT_API __declspec(dllexport) // Note: actually gcc seems to also supports this syntax. - #endif - #else - #ifdef __GNUC__ - #define EXPORT_API __attribute__ ((dllimport)) - #else - #define EXPORT_API __declspec(dllimport) // Note: actually gcc seems to also supports this syntax. - #endif - #endif - #define DLL_LOCAL - #else - #if __GNUC__ >= 4 - #define EXPORT_API __attribute__ ((visibility ("default"))) - #define DLL_LOCAL __attribute__ ((visibility ("hidden"))) - #else - #define EXPORT_API - #define DLL_LOCAL - #endif - #endif -#endif - typedef enum { WAKEUP_SPEECH_STATUS_NONE = -1, /**< None */ WAKEUP_SPEECH_STATUS_BEGINNING_POINT_DETECTED = 1, /**< Beginning point of speech is detected */ @@ -82,6 +56,8 @@ typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH = MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP, } ma_plugin_event_e; typedef enum { @@ -150,6 +126,10 @@ EXPORT_API int wakeup_manager_set_speech_status_callback(wakeup_service_speech_s EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data); +/* Internal API declarations for dependency modules */ + +int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len); + #ifdef __cplusplus } #endif diff --git a/plugins/wakeup-manager/src/dependency_resolver.cpp b/plugins/wakeup-manager/src/dependency_resolver.cpp new file mode 100644 index 0000000..a34d697 --- /dev/null +++ b/plugins/wakeup-manager/src/dependency_resolver.cpp @@ -0,0 +1,224 @@ +/* + * Copyright 2018 Samsung Electronics Co., Ltd + * + * Licensed under the Flora License, Version 1.1 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://floralicense.org/license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "multi_assistant_main.h" +#include "dependency_resolver.h" + +#ifdef LOG_TAG +#undef LOG_TAG +#endif +#define LOG_TAG "dependency-resolver" + +static void *g_handle = NULL; +static mas_dependency g_mas_dependency = { NULL, }; + +int dependency_resolver_initialize(mas_proxy_interface interface) +{ + MAS_LOGD("[Enter]"); + + const int FILEPATH_LEN = 512; + char filepath[FILEPATH_LEN] = {'\0', }; + char *vconf_str = vconf_get_str(MAS_DEPENDENCY_MODULE_PATH); + if (vconf_str) { + snprintf(filepath, FILEPATH_LEN - 1, "%s", vconf_str); + free(vconf_str); + } else { + const char *default_path = MAS_DEPENDENCY_DEFAULT_PATH; + snprintf(filepath, FILEPATH_LEN - 1, "%s/%s", default_path, MAS_DEPENDENCY_DEFAULT_FILENAME); + } + filepath[FILEPATH_LEN - 1] = '\0'; + + char *error; + g_handle = NULL; + g_handle = dlopen(filepath, RTLD_LAZY); + if (NULL != (error = dlerror())) { + MAS_LOGE("[ERROR] Fail to dlopen(%s), error(%s)", filepath, error); + return -1; //MAS_ERROR_OPERATION_FAILED; + } + + g_mas_dependency.initialize = + (mas_dependency_initialize)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_INITIALIZE); + g_mas_dependency.deinitialize = + (mas_dependency_deinitialize)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_DEINITIALIZE); + g_mas_dependency.set_error_callback = + (mas_dependency_set_error_callback)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_SET_ERROR_CALLBACK); + g_mas_dependency.start_recording = + (mas_dependency_start_recording)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_START_RECORDING); + g_mas_dependency.stop_recording = + (mas_dependency_stop_recording)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_STOP_RECORDING); + g_mas_dependency.change_system_volume = + (mas_dependency_change_system_volume)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_CHANGE_SYSTEM_VOLUME); + g_mas_dependency.recover_system_volume = + (mas_dependency_recover_system_volume)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_RECOVER_SYSTEM_VOLUME); + + int ret = -1; + int dependency_version = 0; + if (NULL != g_handle) { + mas_dependency_initialize func = g_mas_dependency.initialize; + + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_INITIALIZE); + } else { + ret = func(interface, &dependency_version); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to initialize, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + MAS_LOGD("g_handle : %p, dependency_version %d", g_handle, dependency_version); + return ret; +} + +int dependency_resolver_deinitialize(void) +{ + MAS_LOGD("g_handle : %p", g_handle); + int ret = -1; + if (NULL != g_handle) { + mas_dependency_deinitialize func = g_mas_dependency.deinitialize; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_DEINITIALIZE); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to deinitialize, ret(%d)", ret); + } + } + + dlclose(g_handle); + g_handle = NULL; + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} + +int dependency_resolver_set_error_callback(mas_dependency_error_cb callback, void* user_data) +{ + int ret = -1; + if (NULL != g_handle) { + mas_dependency_set_error_callback func = g_mas_dependency.set_error_callback; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_SET_ERROR_CALLBACK); + } else { + ret = func(callback, user_data); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set error callback(%p, %p), ret(%d)", callback, user_data, ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + +int dependency_resolver_start_recording(void) +{ + int ret = -1; + MAS_LOGD("g_handle : %p", g_handle); + if (NULL != g_handle) { + mas_dependency_start_recording func = g_mas_dependency.start_recording; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_START_RECORDING); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to start recording, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} + +int dependency_resolver_stop_recording(void) +{ + int ret = -1; + if (NULL != g_handle) { + mas_dependency_stop_recording func = g_mas_dependency.stop_recording; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_STOP_RECORDING); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to stop recording, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} + +int dependency_resolver_change_system_volume() +{ + int ret = -1; + if (NULL != g_handle) { + mas_dependency_change_system_volume func = g_mas_dependency.change_system_volume; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_CHANGE_SYSTEM_VOLUME); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to change system volume, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} + +int dependency_resolver_recover_system_volume() +{ + int ret = -1; + if (NULL != g_handle) { + mas_dependency_recover_system_volume func = g_mas_dependency.recover_system_volume; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_RECOVER_SYSTEM_VOLUME); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to recover system volume, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} diff --git a/plugins/wakeup-manager/src/heap_tracer.cpp b/plugins/wakeup-manager/src/heap_tracer.cpp new file mode 100644 index 0000000..2fc256e --- /dev/null +++ b/plugins/wakeup-manager/src/heap_tracer.cpp @@ -0,0 +1,227 @@ +#include "heap_tracer.h" + +#include +#include + +#include + +#define LOG_TAG "heap_tracer" + +static HeapTracer _tracer; + +const bool RECORD_UNMANAGED_CHUNKS = true; +const int MAX_UNMANAGED_RECORDS = 128; +const bool HEAP_TRACER_LOG_ENABLED = false +; + +#define DESCRIPTION_LENGTH 255 +//#define USE_HEAP_TRACER +//#define RECORD_ALL_HISTORY + +void* vm_calloc(size_t nmemb, size_t size, const char *fmt, ...) +{ + void *ptr = calloc(nmemb, size); +#ifdef USE_HEAP_TRACER + va_list argptr; + va_start(argptr, fmt); + + char description[DESCRIPTION_LENGTH]; + vsnprintf(description, sizeof(description), fmt, argptr); + _tracer.Insert(ptr, description, nmemb * size); + + va_end(argptr); +#endif + return ptr; +} + +void* vm_malloc(size_t size, const char *fmt, ...) +{ + void *ptr = malloc(size); +#ifdef USE_HEAP_TRACER + va_list argptr; + va_start(argptr, fmt); + + char description[DESCRIPTION_LENGTH]; + vsnprintf(description, sizeof(description), fmt, argptr); + _tracer.Insert(ptr, description, size); + + va_end(argptr); +#endif + return ptr; +} + +void vm_free(void *ptr, const char *fmt, ...) +{ + if (ptr) { +#ifdef USE_HEAP_TRACER + va_list argptr; + va_start(argptr, fmt); + + char description[DESCRIPTION_LENGTH]; + vsnprintf(description, sizeof(description), fmt, argptr); + if (false == _tracer.Delete(ptr, description)) { + _tracer.Find(ptr); + } + va_end(argptr); +#endif + free(ptr); + } else { + LOGD("Trying to free a null pointer"); + } +} + +char* vm_strdup(const char *s, const char *fmt, ...) +{ + char *ptr = nullptr; + if (s) { + ptr = strdup(s); +#ifdef USE_HEAP_TRACER + va_list argptr; + va_start(argptr, fmt); + + char description[DESCRIPTION_LENGTH]; + vsnprintf(description, sizeof(description), fmt, argptr); + _tracer.Insert(ptr, description, strlen(s) + 1); + + va_end(argptr); +#endif + } else { + LOGD("Trying to duplicate a null pointer"); + } + return ptr; +} + +void *vm_mark_unmanaged(void *ptr, const char *fmt, ...) +{ +#ifdef USE_HEAP_TRACER + va_list argptr; + va_start(argptr, fmt); + + char description[DESCRIPTION_LENGTH]; + vsnprintf(description, sizeof(description), fmt, argptr); + _tracer.MarkUnmanaged(ptr, description); + + va_end(argptr); +#endif + return ptr; +} + +HeapTracer::HeapTracer() +{ + LOGD("HeapTracer()"); +} + +HeapTracer::~HeapTracer() +{ + LOGD("~HeapTracer()"); + + Trace(); +} + +bool HeapTracer::Insert(void *ptr, std::string description, size_t size) +{ + if (HEAP_TRACER_LOG_ENABLED) { + LOGD("Heap allocated %p [size %d] : %s", ptr, size, description.c_str()); + } + + AllocationEntry entry; + entry.ptr = ptr; + entry.description = description; + entry.size = size; + + std::lock_guard lock(mManagedTableMutex); + mManagedTable[ptr] = entry; + +#ifdef RECORD_ALL_HISTORY + EntryHistory history; + history.entry = entry; + history.operation = EntryOperation::INSERT; + clock_gettime(CLOCK_REALTIME, &(history.ts)); + mEntryHistory.push_back(history); +#endif + return true; +} + +bool HeapTracer::Delete(void *ptr, std::string description) +{ + if (HEAP_TRACER_LOG_ENABLED) { + LOGD("Heap deallocated %p : %s", ptr, description.c_str()); + } + + std::lock_guard lock(mManagedTableMutex); + if (mManagedTable.find(ptr) != mManagedTable.end()) { +#ifdef RECORD_ALL_HISTORY + EntryHistory history; + history.entry = mManagedTable[ptr]; + history.entry.description = description; + history.operation = EntryOperation::DELETE; + clock_gettime(CLOCK_REALTIME, &(history.ts)); + mEntryHistory.push_back(history); +#endif + mManagedTable.erase(ptr); + } else { + LOGE("Heap Trace Error : Trying to delete an entry that does not exist : %p, %s", + ptr, description.c_str()); + return false; + } + return true; +} + +std::string HeapTracer::Find(void *ptr) +{ + std::lock_guard lock(mManagedTableMutex); +#ifdef RECORD_ALL_HISTORY + for (auto entry : mEntryHistory) { + if (entry.entry.ptr == ptr) { + LOGD("Heap history record for %p : %s, %lld.%.9ld, %s", + ptr, + (entry.operation == EntryOperation::INSERT ? "INSERT" : "DELETE"), + (long long)(entry.ts.tv_sec), entry.ts.tv_nsec, + entry.entry.description.c_str()); + } + } +#endif + if (mManagedTable.find(ptr) != mManagedTable.end()) { + return mManagedTable[ptr].description; + } + return std::string("Not found"); +} + +void HeapTracer::MarkUnmanaged(void *ptr, std::string description) +{ + LOGD("Marking as unmanaged : %p %s", ptr, description.c_str()); + + std::lock_guard lock(mManagedTableMutex); + auto iter = mManagedTable.find(ptr); + if (iter != mManagedTable.end()) { + if (RECORD_UNMANAGED_CHUNKS) { + std::lock_guard lock(mUnmanagedRecordsMutex); + if (mUnmanagedRecordsSize >= MAX_UNMANAGED_RECORDS) { + mUnmanagedRecords.pop_front(); + mUnmanagedRecordsSize--; + } + mUnmanagedRecords.push_back(iter->second); + mUnmanagedRecordsSize++; + } + mManagedTable.erase(ptr); + } else { + LOGE("Heap Trace Error : Trying to mark an entry that does not exist as unmanaged : %p", ptr); + } +} + +bool HeapTracer::Empty() +{ + std::lock_guard lock(mManagedTableMutex); + return mManagedTable.empty(); +} + +void HeapTracer::Trace() +{ + LOGD("Heap Trace Start"); + + std::lock_guard lock(mManagedTableMutex); + for (auto item : mManagedTable) { + LOGD("No dealloc information exists for %p [size %d] : %s", item.first, item.second.size, item.second.description.c_str()); + } + LOGD("Heap Trace End"); +} diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 8a16fcd..7e3e305 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -1,21 +1,11 @@ #include "wakeup_audio_manager.h" #include "wakeup_manager_main.h" #include "wakeup_interfaces.h" +#include "dependency_resolver.h" +#include "heap_tracer.h" #include -#ifdef TV_PRODUCT -#include -#include - -#define SMART_CONTROL_EXTEND_CMD 0x03 -#define SMART_CONTROL_START_CMD 0x04 - -static int g_bt_extend_count; -static farfield_voice_h g_farfieldvoice_h = NULL; - -#endif - namespace multiassistant { namespace wakeup @@ -44,333 +34,13 @@ CAudioManager::CAudioManager(IAudioEventObserver *observer) : CAudioManager() subscribe(observer); } -#ifdef TV_PRODUCT -static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data) -{ - CAudioManager *manager = static_cast(user_data); - if (nullptr == manager) return; - - MWR_LOGD("[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address); - return; -} - -static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data) -{ - MWR_LOGD("_bt_hid_audio_data_receive_cb"); - - CAudioManager *manager = static_cast(user_data); - if (nullptr == manager) return; - - static int g_buffer_count = 0; - if (nullptr == voice_data) return; - - if (manager->voice_key_pressed_get()) { - wakeup_speech_data data; - data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; - data.len = voice_data->length; - data.buffer = malloc(voice_data->length); - if (data.buffer) { - memcpy(data.buffer, voice_data->audio_buf, voice_data->length); - manager->add_speech_data(data); - } - } else { - MWR_LOGE("[Recorder ERROR] voice key seems to be already released"); - return; - } - - if (0 == g_buffer_count || 0 == g_buffer_count % 50) { - MWR_LOGD("[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length); - - if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) { - const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 }; - if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) { - MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command"); - } else { - MWR_LOGD("[Recorder] Extend bt audio recorder"); - } - } - g_bt_extend_count++; - - if (100000 == g_buffer_count) { - g_buffer_count = 0; - } - } - - g_buffer_count++; - - return; -} - -static void _ffv_audio_function_cb(void* data, unsigned int length, void* user_data) -{ - CAudioManager *manager = static_cast(user_data); - if (!manager) return; - - /* When voice key is pressed, _bt_hid_audio should receive audio data */ - if (manager->voice_key_pressed_get()) return; - - static int g_buffer_count = 0; - if (0 == g_buffer_count || 0 == g_buffer_count % 50) { - MWR_LOGD("[Recorder INFO] farfield audio function callback is invoked"); - - if (100000 == g_buffer_count) { - g_buffer_count = 0; - } - } - g_buffer_count++; - - long time = get_current_milliseconds_after_epoch(); - - manager->notify_audio_data_recording(time, data, length); - - wakeup_speech_data speech_data; - speech_data.buffer = malloc(length); - if (speech_data.buffer) { - speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; - speech_data.len = length; - memcpy(speech_data.buffer, data, length); - manager->add_background_data(speech_data, time); - } -} -#endif - -static int _apply_device_for_stream_routing(void *user_data) -{ - CAudioManager *manager = static_cast(user_data); - if (nullptr == manager) return -1; - - sound_device_list_h deviceList = NULL; - sound_device_h device = NULL; - sound_device_type_e type; - sound_device_io_direction_e ioDirection; - - if (0 != sound_manager_get_device_list(SOUND_DEVICE_IO_DIRECTION_IN_MASK, &deviceList)) { - MWR_LOGE("[Recorder ERROR] Fail to get current device list"); - return -1; - } - - bool isFound = false; - while (0 == sound_manager_get_next_device(deviceList, &device)) { - if (0 != sound_manager_get_device_type(device, &type)) { - MWR_LOGE("[Recorder ERROR] Fail to get device type"); - continue; - } - if (0 != sound_manager_get_device_io_direction(device, &ioDirection)) { - MWR_LOGE("[Recorder ERROR] Fail to get device io direction"); - continue; - } - - if (SOUND_DEVICE_USB_AUDIO == type && SOUND_DEVICE_IO_DIRECTION_IN == ioDirection) { - if (0 != sound_manager_add_device_for_stream_routing(manager->mStreamInfo, device)) { - MWR_LOGE("[Recorder ERROR] Fail to add device"); - continue; - } - if (0 != sound_manager_apply_stream_routing(manager->mStreamInfo)) { - MWR_LOGE("[Recorder ERROR] Fail to apply stream routing"); - continue; - } - isFound = true; - break; - } - } - - sound_manager_free_device_list(deviceList); - deviceList = NULL; - - if (true != isFound) { - MWR_LOGI("[Recorder] No USB device"); - } else { - MWR_LOGD("[Recorder] Apply device for stream routing"); - } - return 0; -} - -static void _device_connection_changed_cb(sound_device_h device, bool isConnected, void *user_data) -{ - CAudioManager *manager = static_cast(user_data); - if (nullptr == manager) return; - - sound_device_type_e type; - if (isConnected) { - if (0 != sound_manager_get_device_type(device, &type)) { - MWR_LOGE("[Recorder ERROR] Fail to get device type"); - return; - } - if (type == SOUND_DEVICE_USB_AUDIO) { - if (0 != sound_manager_remove_device_for_stream_routing(manager->mStreamInfo, device)) - MWR_LOGE("[Recorder ERROR] Fail to remove device"); - - if (0 != sound_manager_add_device_for_stream_routing(manager->mStreamInfo, device)) { - MWR_LOGE("[Recorder ERROR] Fail to add device"); - return; - } - if (0 != sound_manager_apply_stream_routing(manager->mStreamInfo)) { - MWR_LOGE("[Recorder ERROR} Fail to apply stream routing"); - return; - } - MWR_LOGD("[Recorder] Apply device for stream routing"); - } - } - return; -} - - int CAudioManager::initialize(void) { - const int rate = 16000; - const audio_channel_e channel = AUDIO_CHANNEL_MONO; - const audio_sample_type_e type = AUDIO_SAMPLE_TYPE_S16_LE; - -#ifdef TV_PRODUCT - bool is_bt_failed = false; - - if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) { - MWR_LOGE("[Recorder ERROR] Fail to init bt"); - is_bt_failed = true; - } - - if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, this)) { - MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()"); - is_bt_failed = true; - } - - if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, this)) { - MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()"); - is_bt_failed = true; - } - - if (false == is_bt_failed) { - MWR_LOGD("[Recorder] Bluetooth is available"); - } - - g_farfieldvoice_h = farfield_voice_init(); - if (NULL == g_farfieldvoice_h) { - MWR_LOGE("[Recorder ERROR] Fail to init farfield_voice_init"); - } - - if (g_farfieldvoice_h) { - MWR_LOGI("[Recorder INFO] Register farfield voice audio callback"); - farfield_voice_register_audio_cb(g_farfieldvoice_h, _ffv_audio_function_cb, this); - } -#else - int ret = audio_in_create(rate, channel, type, &mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGE("[Recorder ERROR] Rate(%d) Channel(%d) Type(%d)", rate, channel, type); - MWR_LOGE("[Recorder ERROR] Fail to create audio handle : %d", ret); - return -1; - } - - ret = sound_manager_add_device_connection_changed_cb(SOUND_DEVICE_IO_DIRECTION_IN_MASK, _device_connection_changed_cb, this, &mDeviceId); - if (0 != ret) { - MWR_LOGE("[Recorder ERROR] Fail to add device connection changed callback"); - return -1; - } - - ret = sound_manager_create_stream_information_internal( - SOUND_STREAM_TYPE_VOICE_RECOGNITION_SERVICE, NULL, NULL, &mStreamInfo); - if (0 != ret) { - MWR_LOGE("[Recorder ERROR] Fail to create stream info"); - sound_manager_remove_device_connection_changed_cb(mDeviceId); - audio_in_destroy(mAudioIn); - return -1; - } - - ret = _apply_device_for_stream_routing(this); - if (0 != ret) { - MWR_LOGE("[Recorder ERROR] Fail to apply device for stream routing: %d", ret); - sound_manager_remove_device_connection_changed_cb(mDeviceId); - sound_manager_destroy_stream_information(mStreamInfo); - audio_in_destroy(mAudioIn); - return -1; - } - - ret = audio_in_set_sound_stream_info(mAudioIn, mStreamInfo); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGE("[Recorder ERROR] Fail to set stream info : %d", ret); - sound_manager_remove_device_connection_changed_cb(mDeviceId); - sound_manager_destroy_stream_information(mStreamInfo); - audio_in_destroy(mAudioIn); - return -1; - } - - ret = audio_in_prepare(mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - if (AUDIO_IO_ERROR_SOUND_POLICY == ret) - { - MWR_LOGE("[Recorder ERROR] Audio is busy."); - } else { - MWR_LOGE("[Recorder ERROR] Fail to start audio : %d", ret); - } - sound_manager_remove_device_connection_changed_cb(mDeviceId); - sound_manager_destroy_stream_information(mStreamInfo); - audio_in_destroy(mAudioIn); - return -1; - } - - ret = audio_in_pause(mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGE("[Recorder ERROR] Fail to pause audio in : %d", ret); - } -#endif return 0; } int CAudioManager::deinitialize(void) { - MWR_LOGD("[ENTER]"); - clear_speech_data(); - -#ifdef TV_PRODUCT - if (NULL != g_farfieldvoice_h) { - MWR_LOGD("[Recorder INFO] Unregister farfield voice"); - farfield_voice_unregister_audio_cb(g_farfieldvoice_h); - farfield_voice_final(g_farfieldvoice_h); - g_farfieldvoice_h = NULL; - } - - bt_hid_unset_audio_data_receive_cb(); - bt_hid_host_deinitialize(); - bt_product_deinit(); -#else - int ret = 0; - if (mVirtualSoundStream) { - sound_manager_stop_virtual_stream(mVirtualSoundStream); - ret = sound_manager_destroy_virtual_stream(mVirtualSoundStream); - if (0 != ret) { - MWR_LOGE("[Audio ERROR] Fail to destroy virtual stream, ret(%d)", ret); - } - mVirtualSoundStream = NULL; - } - - if (mDeviceId) { - ret = sound_manager_remove_device_connection_changed_cb(mDeviceId); - if (0 != ret) - MWR_LOGE("[Audio ERROR] Fail to remove device connection changed callback, ret(%d)", ret); - } - - if (mVolumeStream) { - ret = sound_manager_destroy_stream_information(mVolumeStream); - if (0 != ret) { - MWR_LOGE("[Audio ERROR] Fail to destroy stream information, ret(%d)", ret); - } - mVolumeStream = NULL; - } - - ret = audio_in_unprepare(mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGD("[Recorder ERROR] Fail to stop audio : %d", ret); - } - - if (0 != sound_manager_destroy_stream_information(mStreamInfo)) { - MWR_LOGD("[Recorder ERROR] Fail to destroy stream info"); - } - - ret = audio_in_destroy(mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret); - } -#endif - MWR_LOGD("[END]"); return 0; } @@ -387,190 +57,20 @@ void CAudioManager::unsubscribe(IAudioEventObserver *observer) } } -#define FRAME_LENGTH 160 -#define BUFFER_LENGTH FRAME_LENGTH * 2 - -void CAudioManager::recorder_thread_func() -{ -#ifndef TV_PRODUCT - unique_lock lock(mMutex, defer_lock); - static int buffer_count = 0; - - while (!(mStopRecorderThread.load())) { - unsigned char buffer[BUFFER_LENGTH]; - memset(buffer, '\0', BUFFER_LENGTH); - - long time = get_current_milliseconds_after_epoch(); - - int read_bytes = audio_in_read(mAudioIn, buffer, BUFFER_LENGTH); - if (0 > read_bytes) { - LOGE("[Recorder WARNING] Fail to read audio : %d", read_bytes); - break; - } - - notify_audio_data_recording(time, buffer, read_bytes); - - long delta = mBackgroundRecordingDurationMilliseconds; - wakeup_speech_data data; - data.buffer = malloc(read_bytes); - if (data.buffer) { - data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; - data.len = read_bytes; - memcpy(data.buffer, buffer, read_bytes); - add_background_data(data, time); - } - - /* Audio read log */ - if (0 == buffer_count % 300) { - LOGD("[Recorder][%d] Recording... : read_size(%d)", buffer_count, read_bytes); - } - - buffer_count++; - -#ifdef BUF_SAVE_MODE - /* write pcm buffer */ - if (g_pFile) - fwrite(buffer, 1, BUFFER_LENGTH, g_pFile); -#endif - } -#endif -} - void CAudioManager::stop_recording() { - if (mRecorderThread.joinable()) { - MWR_LOGD("mRecorderThread is joinable, trying join()"); - mStopRecorderThread.store(true); - mRecorderThread.join(); - } -#ifndef TV_PRODUCT - int ret = audio_in_pause(mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); - } -#endif + dependency_resolver_stop_recording(); } void CAudioManager::start_recording() { stop_recording(); -#ifdef TV_PRODUCT - /* Do not start normal recorder thread if TV_PRODUCT and mVoiceKeyPressed, - just send bt_hid start message */ - if (mVoiceKeyPressed) { - const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00}; - int bt_retry = 0; - const int max_retry = 5; - while (max_retry > bt_retry) { - int ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data)); - if (BT_ERROR_NONE == ret) { - MWR_LOGD("[Recorder] Start bt audio recorder"); - break; - } else if (BT_ERROR_NOW_IN_PROGRESS == ret) { - MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command : %d", ret); - this_thread::sleep_for(chrono::milliseconds(50)); - bt_retry++; - } else { - break; - } - } - if (max_retry == bt_retry) { - MWR_LOGE("[Recorder ERROR] Fail to start bt audio"); - return; - } - - g_bt_extend_count = 0; - } -#else - int ret = audio_in_resume(mAudioIn); - if (AUDIO_IO_ERROR_NONE != ret) { - MWR_LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); - } - mStopRecorderThread.store(false); - MWR_LOGD("Starting recorder thread"); - mRecorderThread = thread(&CAudioManager::recorder_thread_func, this); -#endif -} - -/* Need to consider adapting conventional producer-consumer model */ -void CAudioManager::streaming_speech_data_thread_func() -{ - MWR_LOGD("[ENTER]"); - - unique_lock lock(mMutex, defer_lock); - int index = 0; - - while (!(mStopStreamingThread.load())) { - int ret = -1; - int cnt = 0; - - /* get feedback data */ - size_t speech_data_size = 0; - lock.lock(); - speech_data_size = mSpeechData.size(); - lock.unlock(); - if (index >= speech_data_size) { - /* empty queue */ - MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret); - - /* waiting */ - while (1) { - this_thread::sleep_for(chrono::milliseconds(10)); - lock.lock(); - speech_data_size = mSpeechData.size(); - lock.unlock(); - if (index < speech_data_size) { - MWR_LOGI("[INFO] Resume thread"); - break; - } - if (200 < cnt) { - MWR_LOGE("[ERROR] Wrong request, there's no pcm data"); - for (const auto& observer : mObservers) { - if (observer) { - if (!observer->on_streaming_audio_data( - WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) { - LOGE("[Recorder WARNING] One of the observer returned false"); - } - } - } - return; - } - cnt++; - } - MWR_LOGI("[INFO] Finish to wait for new feedback data come"); - - /* resume feedback thread */ - continue; - } - - lock.lock(); - wakeup_speech_data& speech_data = mSpeechData.at(index).data; - for (const auto& observer : mObservers) { - if (observer) { - if (!observer->on_streaming_audio_data( - speech_data.event, speech_data.buffer, speech_data.len)) { - LOGE("[Recorder WARNING] One of the observer returned false"); - } - } - } - lock.unlock(); - - if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { - MWR_LOGI("[INFO] Finish to get and send speech data"); - /* Now move all the speech data to previous speech data for later use */ - lock.lock(); - mPreviousSpeechData = move(mSpeechData); - lock.unlock(); - break; - } - - index++; - } + dependency_resolver_start_recording(); } /* Need to consider adapting conventional producer-consumer model */ -void CAudioManager::streaming_previous_speech_data_thread_func() +void CAudioManager::streaming_previous_audio_data_thread_func() { MWR_LOGD("[ENTER]"); @@ -581,19 +81,19 @@ void CAudioManager::streaming_previous_speech_data_thread_func() int cnt = 0; /* get feedback data */ - size_t speech_data_size = 0; + size_t audio_data_size = 0; lock.lock(); - speech_data_size = mPreviousSpeechData.size(); + audio_data_size = mPreviousAudioData.size(); lock.unlock(); /* waiting */ - while (0 >= speech_data_size) { + while (0 >= audio_data_size) { /* empty queue */ MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret); this_thread::sleep_for(chrono::milliseconds(10)); lock.lock(); - speech_data_size = mSpeechData.size(); + audio_data_size = mPreviousAudioData.size(); lock.unlock(); - if (0 < speech_data_size) { + if (0 < audio_data_size) { MWR_LOGI("[INFO] Resume thread"); break; } @@ -602,8 +102,8 @@ void CAudioManager::streaming_previous_speech_data_thread_func() MWR_LOGI("[INFO] Finish to wait for new feedback data come"); lock.lock(); - for (int index = 0; index < speech_data_size; index++) { - wakeup_speech_data& speech_data = mPreviousSpeechData.at(index).data; + for (int index = 0; index < audio_data_size; index++) { + wakeup_speech_data& speech_data = mPreviousAudioData.at(index).data; for (const auto& observer : mObservers) { if (observer) { if (!observer->on_streaming_audio_data( @@ -620,22 +120,23 @@ void CAudioManager::streaming_previous_speech_data_thread_func() } lock.unlock(); } + MWR_LOGD("[EXIT]"); } -void CAudioManager::streaming_background_data_thread_func(long start_time) +void CAudioManager::streaming_audio_data_thread_func(long start_time) { MWR_LOGD("[ENTER]"); unique_lock lock(mMutex, defer_lock); lock.lock(); - auto lead = mBackgroundData.begin(); + auto lead = mAudioData.begin(); auto iter = lead; - while (lead != mBackgroundData.end() && lead->time < start_time) { + while (lead != mAudioData.end() && lead->time < start_time) { iter = lead; advance(lead, 1); } - MWR_LOGD("data_count : %zu", mBackgroundData.size()); + MWR_LOGD("data_count : %zu", mAudioData.size()); lock.unlock(); while (!(mStopStreamingThread.load())) { @@ -644,7 +145,7 @@ void CAudioManager::streaming_background_data_thread_func(long start_time) /* get feedback data */ lock.lock(); - auto end = mBackgroundData.end(); + auto end = mAudioData.end(); lock.unlock(); if (lead == end) { /* empty queue */ @@ -654,8 +155,8 @@ void CAudioManager::streaming_background_data_thread_func(long start_time) while (1) { this_thread::sleep_for(chrono::milliseconds(10)); lock.lock(); - end = mBackgroundData.end(); - auto begin = mBackgroundData.begin(); + end = mAudioData.end(); + auto begin = mAudioData.begin(); lock.unlock(); if (iter == end) { iter = begin; @@ -678,6 +179,7 @@ void CAudioManager::streaming_background_data_thread_func(long start_time) } } } + MWR_LOGD("[EXIT]"); return; } cnt++; @@ -689,10 +191,8 @@ void CAudioManager::streaming_background_data_thread_func(long start_time) } iter = lead; - /* Extracted background data will be used as previous utterance data*/ - lock.lock(); - mSpeechData.push_back(*iter); - lock.unlock(); + + /* FIXME : Extracted audio data here should be used as previous audio data*/ wakeup_speech_data& speech_data = iter->data; for (const auto& observer : mObservers) { @@ -706,75 +206,66 @@ void CAudioManager::streaming_background_data_thread_func(long start_time) if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { MWR_LOGI("[INFO] Finish to get and send speech data"); - /* Now move all the speech data to previous speech data for later use */ - lock.lock(); - mPreviousSpeechData = move(mSpeechData); - lock.unlock(); break; } advance(lead, 1); } + MWR_LOGD("[EXIT]"); } -void CAudioManager::add_speech_data(wakeup_speech_data& speech_data) +void CAudioManager::add_audio_data(wakeup_speech_data& data, long time) { - wakeup_speech_data_with_time data; - data.data = speech_data; - data.time = get_current_milliseconds_after_epoch(); + long delta = mAudioRecordingDurationMilliseconds; - lock_guard lock(mMutex); - mSpeechData.push_back(data); -} + notify_audio_data_recording(time, data.buffer, data.len); + + wakeup_speech_data_with_time data_with_time; + data_with_time.data = data; + data_with_time.time = time; -void CAudioManager::clear_speech_data() -{ lock_guard lock(mMutex); - for (const auto &data : mSpeechData) { - if (data.data.buffer) free(data.data.buffer); + /* Pop items only when the streaming is not activated */ + while(false == mAudioData.empty() && mAudioData.front().time < time - delta) { + const auto &front = mAudioData.front(); + if (front.data.buffer) { + vm_free_simple(front.data.buffer); + } + mAudioData.pop_front(); } - mSpeechData.clear(); + mAudioData.push_back(data_with_time); } -void CAudioManager::finalize_speech_data() +void CAudioManager::feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) { - unsigned char final_buffer[2] = {'\0', }; + if (NULL == buffer || 0 == len) return; + wakeup_speech_data speech_data; - speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH; - speech_data.len = sizeof(final_buffer); - speech_data.buffer = malloc(speech_data.len); + speech_data.buffer = vm_malloc_simple(len); if (speech_data.buffer) { - memcpy(speech_data.buffer, final_buffer, speech_data.len); - wakeup_speech_data_with_time data; - data.data = speech_data; - data.time = get_current_milliseconds_after_epoch(); + long time = get_current_milliseconds_after_epoch(); - lock_guard lock(mMutex); - mSpeechData.push_back(data); + speech_data.event = event; + speech_data.len = len; + memcpy(speech_data.buffer, buffer, len); + add_audio_data(speech_data, time); } } -void CAudioManager::add_background_data(wakeup_speech_data& data, long time) +void CAudioManager::finalize_audio_data() { - long delta = mBackgroundRecordingDurationMilliseconds; - - wakeup_speech_data_with_time data_with_time; - data_with_time.data = data; - data_with_time.time = time; - - lock_guard lock(mMutex); + unsigned char final_buffer[2] = {'\0', }; + wakeup_speech_data speech_data; + speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH; + speech_data.len = sizeof(final_buffer); + speech_data.buffer = vm_malloc_simple(speech_data.len); + if (speech_data.buffer) { + long time = get_current_milliseconds_after_epoch(); - if (mBackgroundData.size() > 0) { - while(mBackgroundData.size() > 0 && mBackgroundData.front().time < time - delta) { - const auto &front = mBackgroundData.front(); - if (front.data.buffer) { - free(front.data.buffer); - } - mBackgroundData.pop_front(); - } + memcpy(speech_data.buffer, final_buffer, speech_data.len); + add_audio_data(speech_data, time); } - mBackgroundData.push_back(data_with_time); } void CAudioManager::notify_audio_data_recording(long time, void* data, int len) @@ -789,18 +280,14 @@ void CAudioManager::notify_audio_data_recording(long time, void* data, int len) } } -void CAudioManager::start_streaming_current_utterance_data(bool from_start_time, long start_time) +void CAudioManager::start_streaming_current_utterance_data(long start_time) { if (mStreamingThread.joinable()) { MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread"); return; } - if (from_start_time) { - clear_speech_data(); - mStreamingThread = thread(&CAudioManager::streaming_background_data_thread_func, this, start_time); - } else { - mStreamingThread = thread(&CAudioManager::streaming_speech_data_thread_func, this); - } + lock_guard lock(mMutex); + mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, start_time); } void CAudioManager::stop_streaming_current_utterance_data() @@ -812,9 +299,7 @@ void CAudioManager::stop_streaming_current_utterance_data() } mStopStreamingThread.store(false); - /* Now move all the speech data to previous speech data for later use */ - lock_guard lock(mMutex); - mPreviousSpeechData = move(mSpeechData); + /* FIXME : Need to move all the speech data to previous speech data for later use */ } void CAudioManager::start_streaming_previous_utterance_data() @@ -823,7 +308,7 @@ void CAudioManager::start_streaming_previous_utterance_data() MWR_LOGE("ERROR : mStreamingPreviousThread is joinable, will not start a new thread"); return; } - mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_speech_data_thread_func, this); + mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_audio_data_thread_func, this); } void CAudioManager::stop_streaming_previous_utterance_data() @@ -844,81 +329,14 @@ void CAudioManager::stop_streaming_follow_up_data() { } -void CAudioManager::voice_key_pressed_set(bool pressed) -{ -#ifdef TV_PRODUCT - if (true == mVoiceKeyPressed && false == pressed) { - bt_hid_rc_stop_sending_voice(NULL); - } - if (pressed != mVoiceKeyPressed) { - if (pressed) { - change_system_volume(); - } else { - recover_system_volume(); - } - } -#endif - mVoiceKeyPressed = pressed; -} - -bool CAudioManager::voice_key_pressed_get() -{ - return mVoiceKeyPressed; -} - void CAudioManager::change_system_volume() { - int ret; - - if (!mVolumeStream) { - ret = sound_manager_create_stream_information_internal(SOUND_STREAM_TYPE_VOICE_RECOGNITION_SERVICE, NULL, NULL, &mVolumeStream); - if (0 != ret) { - MWR_LOGD("[Audio] Fail to create stream information, ret(%d)", ret); - return; - } - } - - if (!mVirtualSoundStream) { - ret = sound_manager_create_virtual_stream(mVolumeStream, &mVirtualSoundStream); - if (0 != ret) { - MWR_LOGD("[Audio ERROR] Fail to create virtual stream, ret(%d)", ret); - return; - } - } - if (mVirtualSoundStream) { - ret = sound_manager_start_virtual_stream(mVirtualSoundStream); - if (0 != ret) { - MWR_LOGD("[Audio ERROR] Fail to start virtual stream, ret(%d)", ret); - return; - } - } + dependency_resolver_change_system_volume(); } void CAudioManager::recover_system_volume() { - int ret; - - if (mVirtualSoundStream) { - ret = sound_manager_stop_virtual_stream(mVirtualSoundStream); - if (0 != ret) { - MWR_LOGD("[Audio ERROR] Fail to stop virtual stream, ret(%d)", ret); - } - ret = sound_manager_destroy_virtual_stream(mVirtualSoundStream); - if (0 != ret) { - MWR_LOGD("[Audio ERROR] Fail to destroy virtual stream, ret(%d)", ret); - return; - } - mVirtualSoundStream = NULL; - } - - if (mVolumeStream) { - ret = sound_manager_destroy_stream_information(mVolumeStream); - if (0 != ret) { - MWR_LOGD("[Audio ERROR] Fail to destroy stream information, ret(%d)", ret); - return; - } - mVolumeStream = NULL; - } + dependency_resolver_recover_system_volume(); } } // wakeup diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index d236882..32d226b 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -490,6 +490,7 @@ void CWakeupEngineManager::add_engine(string name, string path) CallbackUserData *data = static_cast(user_data); if (nullptr == data) return; if (nullptr == data->manager) return; + info.wakeup_engine = data->engine_name.c_str(); data->manager->on_wakeup_event(data->engine_name, info); }, &(callback_user_data.back())); } diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 76f9562..103585b 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -17,6 +17,7 @@ #include "wakeup_manager.h" #include "wakeup_manager_main.h" #include "wakeup_policy_default.h" +#include "dependency_resolver.h" #include @@ -25,102 +26,6 @@ namespace multiassistant namespace wakeup { -#ifdef TV_PRODUCT -#define EFL_BETA_API_SUPPORT - -#include -#include -#include - -static Ecore_Event_Handler* _key_down_handler = NULL; -static Ecore_Event_Handler* _key_up_handler = NULL; - -Eina_Bool _key_down_cb(void* data, int type, void* event) -{ - Ecore_Event_Key *ev = (Ecore_Event_Key *) event; - if (ev) { - MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type); - - if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0 ) { - wakeup_manager_send_assistant_specific_command(DEFAULT_ASSISTANT_APPID, "voice_key_pressed"); - } - } - - return ECORE_CALLBACK_DONE; -} - -Eina_Bool _key_up_cb(void* data, int type, void* event) -{ - Ecore_Event_Key *ev = (Ecore_Event_Key *) event; - if (ev) { - MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type); - - if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0) { - wakeup_manager_send_assistant_specific_command(DEFAULT_ASSISTANT_APPID, "voice_key_released"); - } - } - return ECORE_CALLBACK_DONE; -} - -bool _grab_voice_key(void) -{ - Eina_Bool bRet = true; - bRet = ecore_wl2_window_keygrab_set(NULL, KEY_BT_VOICE, 0, 0, 0, ECORE_WL2_WINDOW_KEYGRAB_SHARED); - MWR_LOGD("ecore_wl2_window_keygrab_set ret[%d] [%s]", bRet, KEY_BT_VOICE); - return bRet; -} - -bool _ungrab_voice_key(void) -{ - Eina_Bool bRet = true; - bRet = ecore_wl2_window_keygrab_unset(NULL, KEY_BT_VOICE, 0, 0); - MWR_LOGD("ecore_wl2_window_keygrab_unset ret[%d] [%s]", bRet, KEY_BT_VOICE); - return bRet; -} - -bool _add_key_cb() -{ - if (_key_down_handler == NULL) - { - MWR_LOGE("_key_down_handler"); - _key_down_handler = ecore_event_handler_add(ECORE_EVENT_KEY_DOWN, _key_down_cb, NULL); - if(_key_down_handler == NULL) - { - MWR_LOGE("_key_down_handler == NULL "); - } - } - - if (_key_up_handler == NULL) - { - MWR_LOGE("_key_down_handler"); - _key_up_handler = ecore_event_handler_add(ECORE_EVENT_KEY_UP, _key_up_cb, NULL); - if(_key_up_handler == NULL) - { - MWR_LOGE("_key_up_handler == NULL "); - } - } - return true; -} - -bool _delete_key_cb(void) -{ - MWR_LOGE("start"); - if (_key_down_handler != NULL) - { - ecore_event_handler_del(_key_down_handler); - _key_down_handler = NULL; - } - - if (_key_up_handler != NULL) - { - ecore_event_handler_del(_key_up_handler); - _key_up_handler = NULL; - } - MWR_LOGE("end"); - return true; -} -#endif - static bool check_language_valid(string language) { return true; @@ -136,6 +41,7 @@ static bool initialize_wakeup_event_info(wakeup_event_info* info) info->wakeup_word = nullptr; info->wakeup_language = nullptr; info->wakeup_voice_id = nullptr; + info->wakeup_engine = nullptr; info->wakeup_confidence_score = 0.0f; info->wakeup_start_time = 0; @@ -200,18 +106,11 @@ bool CWakeupManager::initialize() mWakeupEngineManager.subscribe(&mEngineEventObserver); mWakeupEngineManager.initialize(); -#ifdef TV_PRODUCT - Ecore_Wl2_Display *_ecore_wl2_display = NULL; - - Eina_Bool bRet = ecore_wl2_init(); - MWR_LOGD("ecore_wl2_init: %d", bRet); + mas_proxy_interface interface; + interface.process_event = wakeup_manager_process_event; + interface.feed_audio_data = wakeup_manager_feed_audio_data; - _ecore_wl2_display = ecore_wl2_display_connect(NULL); - MWR_LOGD("_ecore_wl2_display: %p", _ecore_wl2_display); - - _grab_voice_key(); - _add_key_cb(); -#endif + dependency_resolver_initialize(interface); MWR_LOGD("[END]"); return true; @@ -221,10 +120,7 @@ bool CWakeupManager::deinitialize() { MWR_LOGD("[ENTER]"); -#ifdef TV_PRODUCT - _delete_key_cb(); - _ungrab_voice_key(); -#endif + dependency_resolver_deinitialize(); mWakeupEngineManager.unsubscribe(&mEngineEventObserver); mWakeupEngineManager.deinitialize(); @@ -363,6 +259,15 @@ bool CWakeupManager::set_streaming_mode(STREAMING_MODE mode) bool CWakeupManager::change_manager_state(wakeup_manager_state_e state) { + if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState || + WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState) { + if (WAKEUP_MANAGER_STATE_VOICE_FEEDBACK == state || + WAKEUP_MANAGER_STATE_LISTENING == state) { + stop_streaming_utterance_data(); + stop_streaming_previous_utterance_data(); + stop_streaming_follow_up_data(); + } + } mWakeupManagerState = state; mWakeupEngineManager.update_manager_state(state); return true; @@ -396,6 +301,7 @@ bool CWakeupManager::send_assistant_specific_command(string appid, string comman { MWR_LOGD("[ENTER]"); + /* static const string voice_key_pressed{"voice_key_pressed"}; static const string voice_key_released{"voice_key_released"}; @@ -404,6 +310,7 @@ bool CWakeupManager::send_assistant_specific_command(string appid, string comman } else if (0 == command.compare(voice_key_released)) { process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0); } + */ mWakeupEngineManager.engine_set_assistant_specific_command(appid, command); @@ -428,55 +335,79 @@ bool CWakeupManager::change_system_volume(string appid, int event) bool CWakeupManager::update_result_state(string appid, int state) { MWR_LOGD("[ENTER]"); - if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState) { - change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); + if (0 == state) { + if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState || + WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { + change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); + } } MWR_LOGD("[END]"); return true; } +static long get_current_milliseconds_after_epoch() +{ + auto now = chrono::system_clock::now(); + auto now_ms = chrono::time_point_cast(now); + /* number of milliseconds since the epoch of system_clock */ + auto value = now_ms.time_since_epoch(); + + return value.count(); +} + bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) { - MWR_LOGD("[ENTER]"); + MWR_LOGD("[ENTER] : %d", event); if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManagerState) return false; // LOCK REQUIRED if (MA_PLUGIN_EVENT_VOICE_KEY_PRESSED == event) { if (mVoiceKeyPressed != true) { - /* Clear all existing data */ - mAudioManager.clear_speech_data(); + stop_streaming_utterance_data(); + stop_streaming_previous_utterance_data(); + stop_streaming_follow_up_data(); + change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE); mVoiceKeyPressed = true; - mAudioManager.voice_key_pressed_set(mVoiceKeyPressed); - /* (Re)Start recorder thread using bt hid */ + + /* (Re)Start recorder thread using appropriate recorder */ mAudioManager.start_recording(); /* Wakeup default assistant */ /* TODO: apply conversation timeout for selecting assistant here */ wakeup_event_info event_info; initialize_wakeup_event_info(&event_info); + /* Make sure to use background data */ + event_info.wakeup_time_valid = true; + event_info.wakeup_end_time = get_current_milliseconds_after_epoch(); + event_info.wakeup_engine = WAKEUP_ENGINE_VOICE_KEY; - string appid = mWakeupSettings.get_default_assistant_appid(); + static string appid = mWakeupSettings.get_default_assistant_appid(); event_info.wakeup_appid = appid.c_str(); + set_last_wakeup_event_info(event_info); for (const auto& observer : mObservers) { observer->on_wakeup(event_info); } } - } else if (MA_PLUGIN_EVENT_VOICE_KEY_RELEASED == event) { + } else if (MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH == event) { if (mVoiceKeyPressed != false) { mVoiceKeyPressed = false; - mAudioManager.voice_key_pressed_set(mVoiceKeyPressed); - mAudioManager.finalize_speech_data(); + mAudioManager.finalize_audio_data(); if (mWakeupEngineManager.get_audio_data_required()) { /* Restart recorder thread using standard mic */ + mAudioManager.stop_recording(); mAudioManager.start_recording(); } else { mAudioManager.stop_recording(); } } + } else if (MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP == event) { + if (mVoiceKeyPressed != false) { + mVoiceKeyPressed = false; + } } // UNLOCK REQUIRED @@ -540,16 +471,11 @@ bool CWakeupManager::start_streaming_utterance_data() mStreamingMode = STREAMING_MODE::UTTERANCE; bool streaming_by_manager = true; - /* What if the user pressed voice key but then again immediately releases? */ - if (mVoiceKeyPressed) { - mAudioManager.start_streaming_current_utterance_data(); + if (false == mLastWakeupEventInfo.wakeup_time_valid) { + mWakeupEngineManager.start_streaming_current_utterance_data(); + streaming_by_manager = false; } else { - if(mLastWakeupEventInfo.wakeup_time_valid) { - mAudioManager.start_streaming_current_utterance_data(true, mLastWakeupEventInfo.wakeup_end_time); - } else { - mWakeupEngineManager.start_streaming_current_utterance_data(); - streaming_by_manager = false; - } + mAudioManager.start_streaming_current_utterance_data(mLastWakeupEventInfo.wakeup_end_time); } ecore_thread_main_loop_begin(); @@ -667,6 +593,11 @@ CWakeupSettings* CWakeupManager::get_wakeup_settings() return &mWakeupSettings; } +void CWakeupManager::feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) +{ + mAudioManager.feed_audio_data(event, buffer, len); +} + bool CWakeupManager::CEngineEventObserver::on_wakeup_event(string engine_name, wakeup_event_info info) { MWR_LOGD("[ENTER]"); @@ -737,11 +668,15 @@ bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data( void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info info) { if (nullptr == mWakeupManager) return; - mWakeupManager->set_last_wakeup_event_info(info); + mWakeupManager->stop_streaming_utterance_data(); + mWakeupManager->stop_streaming_previous_utterance_data(); + mWakeupManager->stop_streaming_follow_up_data(); + mWakeupManager->change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE); CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager(); if (nullptr == engine_manager) return; + mWakeupManager->set_last_wakeup_event_info(info); engine_manager->set_selected_wakeup_info(info); vector observers = mWakeupManager->get_observers(); for (const auto& observer : observers) { diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 594494b..89e9af5 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -210,12 +210,14 @@ int wakeup_manager_update_voice_feedback_state(const char* appid, int state) { MWR_LOGD("[ENTER]"); + string appid_string; if (NULL == appid) { MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)", appid); - return -1; + } else { + appid_string = appid; } - g_wakeup_manager.update_voice_feedback_state(string{appid}, state); + g_wakeup_manager.update_voice_feedback_state(appid_string, state); MWR_LOGD("[END]"); return 0; @@ -225,12 +227,15 @@ int wakeup_manager_send_assistant_specific_command(const char* appid, const char { MWR_LOGD("[ENTER]"); + string appid_string; + string command_string; if (NULL == appid || NULL == command) { MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), command(%s)", appid, command); - return -1; } + if (appid) appid_string = appid; + if (command) command_string = command; - g_wakeup_manager.send_assistant_specific_command(string{appid}, string{command}); + g_wakeup_manager.send_assistant_specific_command(appid_string, command_string); MWR_LOGD("[END]"); return 0; @@ -240,12 +245,14 @@ int wakeup_manager_change_system_volume(const char* appid, int event) { MWR_LOGD("[ENTER]"); + string appid_string; if (NULL == appid) { MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)", appid); - return -1; + } else { + appid_string = appid; } - g_wakeup_manager.change_system_volume(string{appid}, event); + g_wakeup_manager.change_system_volume(appid_string, event); MWR_LOGD("[END]"); return 0; @@ -255,11 +262,13 @@ int wakeup_manager_update_result_state(const char* appid, int state) { MWR_LOGD("[ENTER]"); + string appid_string; if (NULL == appid) { MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)", appid); - return -1; + } else { + appid_string = appid; } - g_wakeup_manager.update_result_state(string{appid}, state); + g_wakeup_manager.update_result_state(appid_string, state); MWR_LOGD("[END]"); return 0; @@ -446,6 +455,12 @@ int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* us return 0; } +int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) +{ + g_wakeup_manager.feed_audio_data(event, buffer, len); + return 0; +} + void CWakeupEventObserver::on_wakeup(wakeup_event_info info) { if (NULL != g_wakeup_event_cb) { diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index ca29e43..d7fe03a 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -293,6 +293,8 @@ int mas_client_send_recognition_result(int pid, int result) MAS_LOGE("[ERROR] Fail to send recognition result, ret(%d)", ret); } + multi_assistant_service_plugin_update_result_state(NULL, 0); + return ret; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 9308345..819ab1d 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -708,7 +708,7 @@ int multi_assistant_service_plugin_update_result_state(const char* appid, int st } else { ret = func(appid, state); if (0 != ret) { - MAS_LOGE("[ERROR] Fail to stop recording, ret(%d)", ret); + MAS_LOGE("[ERROR] Fail to update result state, ret(%d)", ret); } } } else { -- 2.34.1