From: Ji-hoon Lee Date: Tue, 16 Apr 2019 01:17:11 +0000 (+0900) Subject: Add support for device-specific audio recording X-Git-Tag: submit/tizen/20190524.115101~4 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F51%2F203551%2F1;p=platform%2Fcore%2Fuifw%2Fwakeup-engine-default.git Add support for device-specific audio recording This kind of device-specific behavior should be extracted into a separated module, need to enhance in the future. Change-Id: I042962cb03aea0dc69040bae3d5231665360ad96 --- diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index d2db142..602abe7 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -29,7 +29,7 @@ BuildRequires: pkgconfig(vd-win-util) BuildRequires: pkgconfig(capi-network-bluetooth) BuildRequires: pkgconfig(capi-network-bluetooth-tv) #BuildRequires: pkgconfig(msfapi) -#BuildRequires: pkgconfig(farfield-voice-api) +BuildRequires: pkgconfig(farfield-voice-api) %endif diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 69e8f36..55fccc8 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -30,6 +30,7 @@ IF("${_TV_PRODUCT}" STREQUAL "TRUE") capi-network-bluetooth capi-network-bluetooth-tv vd-win-util + farfield-voice-api ) ENDIF() diff --git a/plugins/wakeup-manager/inc/wakeup_audio_manager.h b/plugins/wakeup-manager/inc/wakeup_audio_manager.h index 6de354f..f791b17 100644 --- a/plugins/wakeup-manager/inc/wakeup_audio_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_audio_manager.h @@ -68,6 +68,10 @@ public: void clear_speech_data(); void finalize_speech_data(); + void add_background_data(wakeup_speech_data& data, long time); + + void notify_audio_data_recording(long time, void* data, int len); + void start_streaming_current_utterance_data(bool from_start_time = false, long start_time = 0); void stop_streaming_current_utterance_data(); diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 238028c..7977c2e 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -6,11 +6,13 @@ #ifdef TV_PRODUCT #include +#include #define SMART_CONTROL_EXTEND_CMD 0x03 #define SMART_CONTROL_START_CMD 0x04 static int g_bt_extend_count; +static farfield_voice_h g_farfieldvoice_h = NULL; #endif @@ -19,6 +21,16 @@ namespace multiassistant namespace wakeup { +static long get_current_milliseconds_after_epoch() +{ + auto now = chrono::system_clock::now(); + auto now_ms = chrono::time_point_cast(now); + /* number of milliseconds since the epoch of system_clock */ + auto value = now_ms.time_since_epoch(); + + return value.count(); +} + CAudioManager::CAudioManager() { } @@ -88,6 +100,38 @@ static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void return; } + +static void _ffv_audio_function_cb(void* data, unsigned int length, void* user_data) +{ + CAudioManager *manager = static_cast(user_data); + if (!manager) return; + + /* When voice key is pressed, _bt_hid_audio should receive audio data */ + if (manager->voice_key_pressed_get()) return; + + static int g_buffer_count = 0; + if (0 == g_buffer_count || 0 == g_buffer_count % 50) { + MWR_LOGD("[Recorder INFO] farfield audio function callback is invoked"); + + if (100000 == g_buffer_count) { + g_buffer_count = 0; + } + } + g_buffer_count++; + + long time = get_current_milliseconds_after_epoch(); + + manager->notify_audio_data_recording(time, data, length); + + wakeup_speech_data speech_data; + speech_data.buffer = malloc(length); + if (speech_data.buffer) { + speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; + speech_data.len = length; + memcpy(speech_data.buffer, data, length); + manager->add_background_data(speech_data, time); + } +} #endif int CAudioManager::initialize(void) @@ -96,6 +140,38 @@ int CAudioManager::initialize(void) const audio_channel_e channel = AUDIO_CHANNEL_MONO; const audio_sample_type_e type = AUDIO_SAMPLE_TYPE_S16_LE; +#ifdef TV_PRODUCT + bool is_bt_failed = false; + + if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) { + MWR_LOGE("[Recorder ERROR] Fail to init bt"); + is_bt_failed = true; + } + + if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, this)) { + MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()"); + is_bt_failed = true; + } + + if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, this)) { + MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()"); + is_bt_failed = true; + } + + if (false == is_bt_failed) { + MWR_LOGD("[Recorder] Bluetooth is available"); + } + + g_farfieldvoice_h = farfield_voice_init(); + if (NULL == g_farfieldvoice_h) { + MWR_LOGD("[Recorder ERROR] Fail to init farfield_voice_init"); + } + + if (g_farfieldvoice_h) { + MWR_LOGD("[Recorder INFO] Register farfield voice audio callback"); + farfield_voice_register_audio_cb(g_farfieldvoice_h, _ffv_audio_function_cb, this); + } +#else int ret = audio_in_create(rate, channel, type, &mAudioIn); if (AUDIO_IO_ERROR_NONE != ret) { MWR_LOGD("[Recorder ERROR] Rate(%d) Channel(%d) Type(%d)", rate, channel, type); @@ -130,28 +206,6 @@ int CAudioManager::initialize(void) audio_in_destroy(mAudioIn); return -1; } - -#ifdef TV_PRODUCT - bool is_bt_failed = false; - - if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) { - MWR_LOGE("[Recorder ERROR] Fail to init bt"); - is_bt_failed = true; - } - - if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, this)) { - MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()"); - is_bt_failed = true; - } - - if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, this)) { - MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()"); - is_bt_failed = true; - } - - if (false == is_bt_failed) { - MWR_LOGD("[Recorder] Bluetooth is available"); - } #endif return 0; } @@ -163,11 +217,17 @@ int CAudioManager::deinitialize(void) clear_speech_data(); #ifdef TV_PRODUCT + if (NULL != g_farfieldvoice_h) { + MWR_LOGD("[Recorder INFO] Unregister farfield voice"); + farfield_voice_unregister_audio_cb(g_farfieldvoice_h); + farfield_voice_final(g_farfieldvoice_h); + g_farfieldvoice_h = NULL; + } + bt_hid_unset_audio_data_receive_cb(); bt_hid_host_deinitialize(); bt_product_deinit(); -#endif - +#else int ret = 0; ret = audio_in_unprepare(mAudioIn); if (AUDIO_IO_ERROR_NONE != ret) { @@ -182,7 +242,7 @@ int CAudioManager::deinitialize(void) if (AUDIO_IO_ERROR_NONE != ret) { MWR_LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret); } - +#endif MWR_LOGD("[END]"); return 0; } @@ -200,16 +260,6 @@ void CAudioManager::unsubscribe(IAudioEventObserver *observer) } } -static long get_current_milliseconds_after_epoch() -{ - auto now = chrono::system_clock::now(); - auto now_ms = chrono::time_point_cast(now); - /* number of milliseconds since the epoch of system_clock */ - auto value = now_ms.time_since_epoch(); - - return value.count(); -} - #define FRAME_LENGTH 160 #define BUFFER_LENGTH FRAME_LENGTH * 2 @@ -231,38 +281,16 @@ void CAudioManager::recorder_thread_func() break; } - for (const auto& observer : mObservers) { - if (observer) { - if (!observer->on_recording_audio_data(time, buffer, read_bytes)) { - LOGE("[Recorder WARNING] One of the observer returned false"); - return; - } - } - } + notify_audio_data_recording(time, buffer, read_bytes); - lock.lock(); long delta = mBackgroundRecordingDurationMilliseconds; - if (mBackgroundData.size() > 0) { - while(mBackgroundData.size() > 0 && mBackgroundData.front().time < time - delta) { - const auto &front = mBackgroundData.front(); - if (front.data.buffer) { - free(front.data.buffer); - } - mBackgroundData.pop_front(); - } - } - lock.unlock(); - - wakeup_speech_data_with_time data; - data.data.buffer = malloc(read_bytes); - if (data.data.buffer) { - data.time = time; - data.data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; - data.data.len = read_bytes; - memcpy(data.data.buffer, buffer, read_bytes); - lock.lock(); - mBackgroundData.push_back(data); - lock.unlock(); + wakeup_speech_data data; + data.buffer = malloc(read_bytes); + if (data.buffer) { + data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; + data.len = read_bytes; + memcpy(data.buffer, buffer, read_bytes); + add_background_data(data, time); } /* Audio read log */ @@ -297,8 +325,7 @@ void CAudioManager::start_recording() #ifdef TV_PRODUCT /* Do not start normal recorder thread if TV_PRODUCT and mVoiceKeyPressed, just send bt_hid start message */ - if (mVoiceKeyPressed) - { + if (mVoiceKeyPressed) { const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00}; int bt_retry = 0; const int max_retry = 5; @@ -322,13 +349,11 @@ void CAudioManager::start_recording() g_bt_extend_count = 0; } - else +#else + mStopRecorderThread.store(false); + MWR_LOGD("Starting recorder thread"); + mRecorderThread = thread(&CAudioManager::recorder_thread_func, this); #endif - { - mStopRecorderThread.store(false); - MWR_LOGD("Starting recorder thread"); - mRecorderThread = thread(&CAudioManager::recorder_thread_func, this); - } } /* Need to consider adapting conventional producer-consumer model */ @@ -540,6 +565,40 @@ void CAudioManager::finalize_speech_data() } } +void CAudioManager::add_background_data(wakeup_speech_data& data, long time) +{ + long delta = mBackgroundRecordingDurationMilliseconds; + + wakeup_speech_data_with_time data_with_time; + data_with_time.data = data; + data_with_time.time = time; + + lock_guard lock(mMutex); + + if (mBackgroundData.size() > 0) { + while(mBackgroundData.size() > 0 && mBackgroundData.front().time < time - delta) { + const auto &front = mBackgroundData.front(); + if (front.data.buffer) { + free(front.data.buffer); + } + mBackgroundData.pop_front(); + } + } + mBackgroundData.push_back(data_with_time); +} + +void CAudioManager::notify_audio_data_recording(long time, void* data, int len) +{ + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_recording_audio_data(time, data, len)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + return; + } + } + } +} + void CAudioManager::start_streaming_current_utterance_data(bool from_start_time, long start_time) { if (mStreamingThread.joinable()) {