WAKEUP_SPEECH_STREAMING_EVENT_START = 1, /**< Start event */
WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE = 2, /**< Continue event */
WAKEUP_SPEECH_STREAMING_EVENT_FINISH = 3 /**< Finish event */
-} wakeup_service_speech_streaming_event_e;
+} wakeup_speech_streaming_event_e;
typedef enum {
WAKEUP_SPEECH_STATUS_NONE = -1, /**< None */
WAKEUP_SPEECH_STATUS_BEGINNING_POINT_DETECTED = 1, /**< Beginning point of speech is detected */
WAKEUP_SPEECH_STATUS_END_POINT_DETECTED = 2 /**< End point of speech is detected */
-} wakeup_service_speech_status_e;
+} wakeup_speech_status_e;
typedef enum {
WAKEUP_ASR_RESULT_EVENT_FINAL = 0, /**< Event when either the full matched or the final result is delivered */
typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info event, const char* wakeup_word, void* user_data);
-typedef void (*wakeup_service_speech_streaming_cb)(wakeup_service_speech_streaming_event_e event, unsigned char* buffer, int len, void *user_data);
+typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event_e event, unsigned char* buffer, int len, void *user_data);
-typedef void (*wakeup_service_speech_status_cb)(wakeup_service_speech_status_e status, void *user_data);
+typedef void (*wakeup_service_speech_status_cb)(wakeup_speech_status_e status, void *user_data);
typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data);
src/wakeup_manager.cpp
src/wakeup_policy.cpp
src/wakeup_policy_default.cpp
- src/wakeup_manager_audio.cpp
+ src/wakeup_audio_manager.cpp
)
FOREACH(flag ${wmpkgs_CFLAGS})
--- /dev/null
+/*
+ * Copyright 2018 Samsung Electronics Co., Ltd
+ *
+ * Licensed under the Flora License, Version 1.1 (the License);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://floralicense.org/license/
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef _WAKEUP_AUDIO_MANAGER_H_
+#define _WAKEUP_AUDIO_MANAGER_H_
+
+#include "wakeup_interfaces.h"
+
+#include <atomic>
+#include <thread>
+#include <vector>
+
+#include <audio_io.h>
+#include <sound_manager.h>
+
+namespace multiassistant
+{
+namespace wakeup
+{
+
+class IAudioDataObserver
+{
+public:
+ virtual ~IAudioDataObserver() {}
+ virtual bool on_recording_audio_data(long time, void* data, int len) = 0;
+ virtual bool on_streaming_audio_data(
+ wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0;
+};
+
+class CAudioManager
+{
+public:
+ CAudioManager();
+ ~CAudioManager();
+
+ CAudioManager(const CAudioManager&) = delete;
+ CAudioManager& operator=(const CAudioManager&) = delete;
+
+ int initialize();
+ int deinitialize();
+
+ void subscribe(IAudioDataObserver *observer);
+ void unsubscribe(IAudioDataObserver *observer);
+
+ void start_recording();
+ void stop_recording();
+
+ void clear_audio_data();
+ void finalize_audio_data();
+
+ void start_streaming();
+ void stop_streaming();
+
+ void voice_key_pressed_set(bool pressed);
+ bool voice_key_pressed_get();
+
+ void add_custom_speech_data(wakeup_speech_data& data);
+private:
+ void recorder_thread_func(void);
+ void streaming_thread_func(void);
+
+ std::vector<IAudioDataObserver*> mObservers;
+
+ audio_in_h mAudioIn{nullptr};
+ sound_stream_info_h mStreamInfo{nullptr};
+
+ std::thread mRecorderThread;
+ std::atomic_bool mStopRecorderThread{false};
+
+ std::thread mStreamingThread;
+ std::atomic_bool mStopStreamingThread{false};
+
+ std::vector<wakeup_speech_data> mSpeechData;
+ bool mVoiceKeyPressed{false};
+};
+
+} // wakeup
+} // multiassistant
+
+#endif /* _WAKEUP_AUDIO_MANAGER_H_ */
const char *extra_data_description;
} wakeup_event_info;
+typedef enum {
+ WAKEUP_SPEECH_STREAMING_EVENT_FAIL = -1, /**< Failed */
+ WAKEUP_SPEECH_STREAMING_EVENT_START = 1, /**< Start event */
+ WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE = 2, /**< Continue event */
+ WAKEUP_SPEECH_STREAMING_EVENT_FINISH = 3 /**< Finish event */
+} wakeup_speech_streaming_event_e;
+
+typedef struct {
+ wakeup_speech_streaming_event_e event;
+ void* buffer;
+ int len;
+} wakeup_speech_data;
+
#endif /* _WAKEUP_INTERFACES_H_ */
\ No newline at end of file
#endif
typedef enum {
- WAKEUP_SPEECH_STREAMING_EVENT_FAIL = -1, /**< Failed */
- WAKEUP_SPEECH_STREAMING_EVENT_START = 1, /**< Start event */
- WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE = 2, /**< Continue event */
- WAKEUP_SPEECH_STREAMING_EVENT_FINISH = 3 /**< Finish event */
-} wakeup_service_speech_streaming_event_e;
-
-typedef enum {
WAKEUP_SPEECH_STATUS_NONE = -1, /**< None */
WAKEUP_SPEECH_STATUS_BEGINNING_POINT_DETECTED = 1, /**< Beginning point of speech is detected */
WAKEUP_SPEECH_STATUS_END_POINT_DETECTED = 2 /**< End point of speech is detected */
typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info info, void* user_data);
-typedef void (*wakeup_service_speech_streaming_cb)(wakeup_service_speech_streaming_event_e event, void* buffer, int len, void *user_data);
+typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event_e event, void* buffer, int len, void *user_data);
typedef void (*wakeup_service_speech_status_cb)(wakeup_service_speech_status_e status, void *user_data);
*************************************************************************************/
#define MA_WAKEUP_ENGINE_PATH tzplatform_mkpath(tzplatform_getid("TZ_SYS_RO_SHARE"), "multiassistant/engines")
-typedef struct {
- wakeup_service_speech_streaming_event_e event;
- void* buffer;
- int len;
-} wakeup_engine_speech_data;
-
typedef enum {
MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0,
MA_PLUGIN_EVENT_VOICE_KEY_RELEASED,
#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT "wakeup_engine_get_utterance_data_count"
typedef int (*wakeup_engine_get_utterance_data_count)(void);
#define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA "wakeup_engine_get_utterance_data"
-typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_engine_speech_data *data);
+typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_speech_data *data);
#define MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND "wakeup_engine_set_assistant_specific_command"
typedef int (*wakeup_engine_set_assistant_specific_command)(const char* appid, const char* command);
#define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_engine_set_wakeup_event_callback"
+++ /dev/null
-/*
- * Copyright 2018 Samsung Electronics Co., Ltd
- *
- * Licensed under the Flora License, Version 1.1 (the License);
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://floralicense.org/license/
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-#ifndef _WAKEUP_MANAGER_AUDIO_H_
-#define _WAKEUP_MANAGER_AUDIO_H_
-
-int wakeup_manager_audio_initialize(void);
-int wakeup_manager_audio_deinitialize(void);
-
-#endif /* _WAKEUP_MANAGER_AUDIO_H_ */
#include "wakeup_interfaces.h"
+namespace multiassistant
+{
+namespace wakeup
+{
+
class IWakeupEventObserver
{
public:
std::unique_ptr<CWakeupPolicyImpl> mImpl;
};
+} // wakeup
+} // multiassistant
+
#endif /* _WAKEUP_POLICY_H_ */
#include <Ecore.h>
+namespace multiassistant
+{
+namespace wakeup
+{
+
class CWakeupPolicyDefault : public CWakeupPolicy
{
public:
Ecore_Timer *mTimer{nullptr};
};
+} // wakeup
+} // multiassistant
+
#endif /* _WAKEUP_POLICY_H_ */
#include <string>
#include <vector>
+namespace multiassistant
+{
+namespace wakeup
+{
+
class CWakeupPolicyImpl
{
public:
std::vector<IWakeupEventObserver*> mObservers;
};
+} // wakeup
+} // multiassistant
+
#endif /* _WAKEUP_POLICY_IMPL_H_ */
--- /dev/null
+#include "wakeup_audio_manager.h"
+#include "wakeup_manager_main.h"
+#include "wakeup_interfaces.h"
+
+#include <algorithm>
+
+#ifdef TV_PRODUCT
+#include <bluetooth_product.h>
+
+#define SMART_CONTROL_EXTEND_CMD 0x03
+#define SMART_CONTROL_START_CMD 0x04
+
+static int g_bt_extend_count;
+
+#endif
+
+namespace multiassistant
+{
+namespace wakeup
+{
+
+CAudioManager::CAudioManager()
+{
+}
+
+CAudioManager::~CAudioManager()
+{
+}
+
+#ifdef TV_PRODUCT
+static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data)
+{
+ CAudioManager *manager = static_cast<CAudioManager*>(user_data);
+ if (nullptr == manager) return;
+
+ MWR_LOGD("[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address);
+ return;
+}
+
+static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data)
+{
+ CAudioManager *manager = static_cast<CAudioManager*>(user_data);
+ if (nullptr == manager) return;
+
+ static int g_buffer_count = 0;
+ if (nullptr == voice_data) return;
+
+ if (manager->voice_key_pressed_get()) {
+ wakeup_speech_data data;
+ data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE;
+ data.len = voice_data->length;
+ data.buffer = malloc(voice_data->length);
+ if (data.buffer) {
+ memcpy(data.buffer, voice_data->audio_buf, voice_data->length);
+ manager->add_custom_speech_data(data);
+ }
+ } else {
+ MWR_LOGE("[Recorder ERROR] voice key seems to be already released");
+ return;
+ }
+
+ if (0 == g_buffer_count || 0 == g_buffer_count % 50) {
+ MWR_LOGD("[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length);
+
+ if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) {
+ const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 };
+ if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command");
+ } else {
+ MWR_LOGD("[Recorder] Extend bt audio recorder");
+ }
+ }
+ g_bt_extend_count++;
+
+ if (100000 == g_buffer_count) {
+ g_buffer_count = 0;
+ }
+ }
+
+ g_buffer_count++;
+
+ return;
+}
+#endif
+
+int CAudioManager::initialize(void)
+{
+ const int rate = 16000;
+ const audio_channel_e channel = AUDIO_CHANNEL_MONO;
+ const audio_sample_type_e type = AUDIO_SAMPLE_TYPE_S16_LE;
+
+ int ret = audio_in_create(rate, channel, type, &mAudioIn);
+ if (AUDIO_IO_ERROR_NONE != ret) {
+ MWR_LOGD("[Recorder ERROR] Rate(%d) Channel(%d) Type(%d)", rate, channel, type);
+ MWR_LOGD("[Recorder ERROR] Fail to create audio handle : %d", ret);
+ return -1;
+ }
+
+ if (0 != sound_manager_create_stream_information(
+ SOUND_STREAM_TYPE_VOICE_RECOGNITION, NULL, NULL, &mStreamInfo)) {
+ MWR_LOGD("[Recorder ERROR] Fail to create stream info");
+ audio_in_destroy(mAudioIn);
+ return -1;
+ }
+
+ ret = audio_in_set_sound_stream_info(mAudioIn, mStreamInfo);
+ if (AUDIO_IO_ERROR_NONE != ret) {
+ MWR_LOGD("[Recorder ERROR] Fail to set stream info : %d", ret);
+ sound_manager_destroy_stream_information(mStreamInfo);
+ audio_in_destroy(mAudioIn);
+ return -1;
+ }
+
+ ret = audio_in_prepare(mAudioIn);
+ if (AUDIO_IO_ERROR_NONE != ret) {
+ if (AUDIO_IO_ERROR_SOUND_POLICY == ret)
+ {
+ MWR_LOGD("[Recorder ERROR] Audio is busy.");
+ } else {
+ MWR_LOGD("[Recorder ERROR] Fail to start audio : %d", ret);
+ }
+ sound_manager_destroy_stream_information(mStreamInfo);
+ audio_in_destroy(mAudioIn);
+ return -1;
+ }
+
+#ifdef TV_PRODUCT
+ bool is_bt_failed = false;
+
+ if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) {
+ MWR_LOGE("[Recorder ERROR] Fail to init bt");
+ is_bt_failed = true;
+ }
+
+ if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, this)) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()");
+ is_bt_failed = true;
+ }
+
+ if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, this)) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()");
+ is_bt_failed = true;
+ }
+
+ if (false == is_bt_failed) {
+ MWR_LOGD("[Recorder] Bluetooth is available");
+ }
+#endif
+}
+
+int CAudioManager::deinitialize(void)
+{
+ MWR_LOGD("[ENTER]");
+
+ for (const auto &data : mSpeechData) {
+ free(data.buffer);
+ }
+ mSpeechData.clear();
+
+#ifdef TV_PRODUCT
+ bt_hid_unset_audio_data_receive_cb();
+ bt_hid_host_deinitialize();
+ bt_product_deinit();
+#endif
+
+ int ret = 0;
+ ret = audio_in_unprepare(mAudioIn);
+ if (AUDIO_IO_ERROR_NONE != ret) {
+ MWR_LOGD("[Recorder ERROR] Fail to stop audio : %d", ret);
+ }
+
+ if (0 != sound_manager_destroy_stream_information(mStreamInfo)) {
+ MWR_LOGD("[Recorder ERROR] Fail to destroy stream info");
+ }
+
+ ret = audio_in_destroy(mAudioIn);
+ if (AUDIO_IO_ERROR_NONE != ret) {
+ MWR_LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret);
+ }
+
+ MWR_LOGD("[END]");
+ return 0;
+}
+
+void CAudioManager::subscribe(IAudioDataObserver *observer)
+{
+ mObservers.push_back(observer);
+}
+
+void CAudioManager::unsubscribe(IAudioDataObserver *observer)
+{
+ auto iter = std::find(mObservers.begin(), mObservers.end(), observer);
+ if (iter != mObservers.end()) {
+ mObservers.erase(iter);
+ }
+}
+
+#define FRAME_LENGTH 160
+#define BUFFER_LENGTH FRAME_LENGTH * 2
+
+void CAudioManager::recorder_thread_func()
+{
+ static int buffer_count = 0;
+
+ while (!(mStopRecorderThread.load())) {
+ unsigned char buffer[BUFFER_LENGTH];
+ int ret;
+ memset(buffer, '\0', BUFFER_LENGTH);
+
+ auto now = std::chrono::system_clock::now();
+ auto now_ms = std::chrono::time_point_cast<std::chrono::milliseconds>(now);
+ /* number of milliseconds since the epoch of system_clock */
+ auto value = now_ms.time_since_epoch();
+
+ static long time = 0;
+ if (time == value.count()) {
+ LOGE("[Recorder WARNING] Time value duplicated : %lu", time);
+ }
+ time = value.count();
+
+ int read_bytes = audio_in_read(mAudioIn, buffer, BUFFER_LENGTH);
+ if (0 > read_bytes) {
+ LOGE("[Recorder WARNING] Fail to read audio : %d", read_bytes);
+ break;
+ }
+ // LOCK REQUIRED
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_recording_audio_data(time, buffer, read_bytes)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ return;
+ }
+ }
+ }
+
+ // UNLOCK REQUIRED
+ /* Audio read log */
+ if (0 == buffer_count % 100) {
+ LOGD("[Recorder][%d] Recording... : read_size(%d)", buffer_count, read_bytes);
+ }
+
+ buffer_count++;
+
+#ifdef BUF_SAVE_MODE
+ /* write pcm buffer */
+ if (g_pFile)
+ fwrite(buffer, 1, BUFFER_LENGTH, g_pFile);
+#endif
+ }
+}
+
+void CAudioManager::stop_recording()
+{
+ if (mRecorderThread.joinable()) {
+ MWR_LOGD("mRecorderThread is joinable, trying join()");
+ mStopRecorderThread.store(true);
+ mRecorderThread.join();
+ }
+}
+
+void CAudioManager::start_recording()
+{
+ stop_recording();
+
+#ifdef TV_PRODUCT
+ /* Do not start normal recorder thread if TV_PRODUCT and mVoiceKeyPressed,
+ just send bt_hid start message */
+ if (mVoiceKeyPressed)
+ {
+ const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00};
+ int bt_retry = 0;
+ const int max_retry = 5;
+ while (max_retry > bt_retry) {
+ int ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data));
+ if (BT_ERROR_NONE == ret) {
+ MWR_LOGD("[Recorder] Start bt audio recorder");
+ break;
+ } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command : %d", ret);
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ bt_retry++;
+ } else {
+ break;
+ }
+ }
+ if (max_retry == bt_retry) {
+ MWR_LOGE("[Recorder ERROR] Fail to start bt audio");
+ return;
+ }
+
+ g_bt_extend_count = 0;
+ }
+ else
+#endif
+ {
+ mStopRecorderThread.store(false);
+ mRecorderThread = std::thread(&CAudioManager::recorder_thread_func, this);
+ }
+}
+
+
+void CAudioManager::streaming_thread_func(void)
+{
+ MWR_LOGD("[ENTER]");
+
+ MWR_LOGD("data_count : %zu", mSpeechData.size());
+
+ int index = 0;
+
+ while (1) {
+ int ret = -1;
+ int cnt = 0;
+
+ /* get feedback data */
+ if (index >= mSpeechData.size()) {
+ /* empty queue */
+ MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
+
+ /* waiting */
+ while (1) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ if (index < mSpeechData.size()) {
+ MWR_LOGI("[INFO] Resume thread");
+ break;
+ }
+ if (200 < cnt) {
+ MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_streaming_audio_data(
+ WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+ return;
+ }
+ cnt++;
+ }
+ MWR_LOGI("[INFO] Finish to wait for new feedback data come");
+
+ /* resume feedback thread */
+ continue;
+ }
+
+ wakeup_speech_data &speech_data = mSpeechData.at(index);
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_streaming_audio_data(
+ speech_data.event, speech_data.buffer, speech_data.len)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+
+ if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
+ MWR_LOGI("[INFO] Finish to get and send speech data");
+ break;
+ }
+
+ index++;
+ }
+}
+
+void CAudioManager::clear_audio_data()
+{
+ for (const auto &speech_data : mSpeechData) {
+ if (speech_data.buffer) free(speech_data.buffer);
+ }
+ mSpeechData.clear();
+}
+
+void CAudioManager::finalize_audio_data()
+{
+ unsigned char final_buffer[2] = {'\0', };
+ wakeup_speech_data speech_data;
+ speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH;
+ speech_data.len = sizeof(final_buffer);
+ speech_data.buffer = malloc(speech_data.len);
+ if (speech_data.buffer) {
+ memcpy(speech_data.buffer, final_buffer, speech_data.len);
+ mSpeechData.push_back(speech_data);
+ }
+}
+
+void CAudioManager::start_streaming()
+{
+ mStreamingThread = std::thread(&CAudioManager::streaming_thread_func, this);
+}
+
+void CAudioManager::stop_streaming()
+{
+ if (mStreamingThread.joinable()) {
+ MWR_LOGD("mStreamingThread is joinable, trying join()");
+ mStopStreamingThread.store(true);
+ mStreamingThread.join();
+ }
+ mStopStreamingThread.store(false);
+}
+
+void CAudioManager::voice_key_pressed_set(bool pressed)
+{
+#ifdef TV_PRODUCT
+ if (true == mVoiceKeyPressed && false == pressed) {
+ bt_hid_rc_stop_sending_voice(NULL);
+ }
+#endif
+ mVoiceKeyPressed = pressed;
+}
+
+bool CAudioManager::voice_key_pressed_get()
+{
+ return mVoiceKeyPressed;
+}
+
+void CAudioManager::add_custom_speech_data(wakeup_speech_data& data)
+{
+ mSpeechData.push_back(data);
+}
+
+} // wakeup
+} // multiassistant
#include <memory>
#include <algorithm>
-#include <audio_io.h>
-#include <sound_manager.h>
#include <vconf.h>
#include "wakeup_manager_main.h"
#include "wakeup_manager.h"
-#include "wakeup_manager_audio.h"
+#include "wakeup_audio_manager.h"
#include "wakeup_policy_default.h"
+#ifdef TV_PRODUCT
+#include <Ecore_Input.h>
+
+#define EFL_BETA_API_SUPPORT
+
+#include <Ecore_Wl2.h>
+#include <Key_Mode.h>
+
+Ecore_Event_Handler* _key_down_handler = NULL;
+Ecore_Event_Handler* _key_up_handler = NULL;
+
+#endif
+
+static bool g_audio_data_required = false;
+bool g_voice_key_pressed = false;
+
+static std::thread g_engine_data_thread;
+static std::atomic_bool g_engine_data_thread_should_stop;
+
static wakeup_service_wakeup_event_cb g_wakeup_event_cb;
static void* g_wakeup_event_user_data;
#define MAX_WAKEUP_ENGINE_NUM 10
static int g_engine_count = 0;
-#ifdef TV_PRODUCT
-#define TV_BT_MODE
-#include <bluetooth_product.h>
-#endif
-
typedef struct {
bool active{false};
bool enabled{false};
static char* g_current_language = NULL;
static wakeup_manager_state_e g_wakeup_manager_state;
-static bool g_audio_data_required = false;
-static audio_in_h g_audio_h = NULL;
-static sound_stream_info_h g_stream_info_h = NULL;
-
-static std::thread g_recorder_thread;
-static std::atomic_bool g_recorder_thread_should_stop;
-
-static std::thread g_speech_data_thread;
-static std::atomic_bool g_speech_data_thread_should_stop;
-
-static bool g_voice_key_pressed = false;
-static std::vector<wakeup_engine_speech_data> g_speech_data;
-
#define DEFAULT_ASSISTANT_APPID "com.samsung.bixby-voice"
#define WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid"
static wakeup_settings g_wakeup_settings;
-class CWakeupEventObserver : public IWakeupEventObserver
+class CWakeupEventObserver : public multiassistant::wakeup::IWakeupEventObserver
{
public:
void on_wakeup(wakeup_event_info info) override;
};
-static std::unique_ptr<CWakeupPolicy> g_wakeup_policy;
+static std::unique_ptr<multiassistant::wakeup::CWakeupPolicy> g_wakeup_policy;
static CWakeupEventObserver g_wakeup_event_observer;
-#ifdef TV_BT_MODE
-
-#define EFL_BETA_API_SUPPORT
-
-#include <Ecore_Wl2.h>
-#include <Key_Mode.h>
-#include <Ecore_Input.h>
-
-static int g_bt_extend_count;
-
-#define SMART_CONTROL_EXTEND_CMD 0x03
-#define SMART_CONTROL_START_CMD 0x04
-
-Ecore_Event_Handler* _key_down_handler = NULL;
-Ecore_Event_Handler* _key_up_handler = NULL;
+class CAudioDataObserver : public multiassistant::wakeup::IAudioDataObserver
+{
+public:
+ bool on_recording_audio_data(long time, void* data, int len) override;
+ bool on_streaming_audio_data(
+ wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override;
+};
+static multiassistant::wakeup::CAudioManager g_audio_manager;
+static CAudioDataObserver g_audio_data_observer;
+#ifdef TV_PRODUCT
Eina_Bool _key_down_cb(void* data, int type, void* event)
{
Ecore_Event_Key *ev = (Ecore_Event_Key *) event;
MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type);
if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0) {
- bt_hid_rc_stop_sending_voice(NULL);
wakeup_manager_send_assistant_specific_command(0, "voice_key_released");
}
}
MWR_LOGE("end");
return true;
}
-
-static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data)
-{
- MWR_LOGD("[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address);
- return;
-}
-
-static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data)
-{
- static int g_buffer_count = 0;
- if (nullptr == voice_data) return;
-
- if (g_voice_key_pressed) {
- wakeup_engine_speech_data data;
- data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE;
- data.len = voice_data->length;
- data.buffer = malloc(voice_data->length);
- if (data.buffer) {
- memcpy(data.buffer, voice_data->audio_buf, voice_data->length);
- g_speech_data.push_back(data);
- }
- } else {
- MWR_LOGE("[Recorder ERROR] voice key seems to be already released");
- return;
- }
-
- if (0 == g_buffer_count || 0 == g_buffer_count % 50) {
- MWR_LOGD("[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length);
-
- if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) {
- const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 };
- if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) {
- MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command");
- } else {
- MWR_LOGD("[Recorder] Extend bt audio recorder");
- }
- }
- g_bt_extend_count++;
-
- if (100000 == g_buffer_count) {
- g_buffer_count = 0;
- }
- }
-
- g_buffer_count++;
-
- return;
-}
-#endif
-
-static int recorder_initialize(void)
-{
- const int rate = 16000;
- const audio_channel_e channel = AUDIO_CHANNEL_MONO;
- const audio_sample_type_e type = AUDIO_SAMPLE_TYPE_S16_LE;
-
- int ret = audio_in_create(rate, channel, type, &g_audio_h);
- if (AUDIO_IO_ERROR_NONE != ret) {
- MWR_LOGD("[Recorder ERROR] Rate(%d) Channel(%d) Type(%d)", rate, channel, type);
- MWR_LOGD("[Recorder ERROR] Fail to create audio handle : %d", ret);
- return -1;
- }
-
- if (0 != sound_manager_create_stream_information(
- SOUND_STREAM_TYPE_VOICE_RECOGNITION, NULL, NULL, &g_stream_info_h)) {
- MWR_LOGD("[Recorder ERROR] Fail to create stream info");
- audio_in_destroy(g_audio_h);
- return -1;
- }
-
- ret = audio_in_set_sound_stream_info(g_audio_h, g_stream_info_h);
- if (AUDIO_IO_ERROR_NONE != ret) {
- MWR_LOGD("[Recorder ERROR] Fail to set stream info : %d", ret);
- sound_manager_destroy_stream_information(g_stream_info_h);
- audio_in_destroy(g_audio_h);
- return -1;
- }
-
- ret = audio_in_prepare(g_audio_h);
- if (AUDIO_IO_ERROR_NONE != ret) {
- if (AUDIO_IO_ERROR_SOUND_POLICY == ret)
- {
- MWR_LOGD("[Recorder ERROR] Audio is busy.");
- } else {
- MWR_LOGD("[Recorder ERROR] Fail to start audio : %d", ret);
- }
- sound_manager_destroy_stream_information(g_stream_info_h);
- audio_in_destroy(g_audio_h);
- return -1;
- }
-
-#ifdef TV_BT_MODE
- bool is_bt_failed = false;
-
- if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) {
- MWR_LOGE("[Recorder ERROR] Fail to init bt");
- is_bt_failed = true;
- }
-
- if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, NULL)) {
- MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()");
- is_bt_failed = true;
- }
-
- if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, NULL)) {
- MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()");
- is_bt_failed = true;
- }
-
- if (false == is_bt_failed) {
- MWR_LOGD("[Recorder] Bluetooth is available");
- }
-
- _grab_voice_key();
- _add_key_cb();
-#endif
-}
-
-static int recorder_deinitialize(void)
-{
- MWR_LOGD("[ENTER]");
-
-#ifdef TV_BT_MODE
- _delete_key_cb();
- _ungrab_voice_key();
-
- bt_hid_unset_audio_data_receive_cb();
- bt_hid_host_deinitialize();
- bt_product_deinit();
-#endif
-
- int ret = 0;
- ret = audio_in_unprepare(g_audio_h);
- if (AUDIO_IO_ERROR_NONE != ret) {
- MWR_LOGD("[Recorder ERROR] Fail to stop audio : %d", ret);
- }
-
- if (0 != sound_manager_destroy_stream_information(g_stream_info_h)) {
- MWR_LOGD("[Recorder ERROR] Fail to destroy stream info");
- }
-
- ret = audio_in_destroy(g_audio_h);
- if (AUDIO_IO_ERROR_NONE != ret) {
- MWR_LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret);
- }
-
- MWR_LOGD("[END]");
- return 0;
-}
-
-#define FRAME_LENGTH 160
-#define BUFFER_LENGTH FRAME_LENGTH * 2
-
-static void recorder_thread_func()
-{
- static int buffer_count = 0;
-
- while (!(g_recorder_thread_should_stop.load())) {
- unsigned char buffer[BUFFER_LENGTH];
- int ret;
- memset(buffer, '\0', BUFFER_LENGTH);
-
- auto now = std::chrono::system_clock::now();
- auto now_ms = std::chrono::time_point_cast<std::chrono::milliseconds>(now);
- /* number of milliseconds since the epoch of system_clock */
- auto value = now_ms.time_since_epoch();
-
- static long time = 0;
- if (time == value.count()) {
- LOGE("[Recorder WARNING] Time value duplicated : %lu", time);
- }
- time = value.count();
-
- int read_bytes = audio_in_read(g_audio_h, buffer, BUFFER_LENGTH);
- if (0 > read_bytes) {
- LOGE("[Recorder WARNING] Fail to read audio : %d", read_bytes);
- break;
- }
- // LOCK REQUIRED
- if (!g_audio_data_required) return;
- for (int loop = 0;loop < g_engine_count;loop++) {
- if (g_wakeup_engine_info[loop].audio_data_require_status &&
- g_wakeup_engine_info[loop].interface.feed_audio_data) {
- ret = g_wakeup_engine_info[loop].interface.feed_audio_data(time, buffer, read_bytes);
- if (0 == ret) {
- LOGE("[ERROR] Fail to feed speech data, ret(%d)", ret);
- }
- }
- }
- // UNLOCK REQUIRED
- /* Audio read log */
- if (0 == buffer_count % 100) {
- LOGD("[Recorder][%d] Recording... : read_size(%d)", buffer_count, read_bytes);
- }
-
- buffer_count++;
-
-#ifdef BUF_SAVE_MODE
- /* write pcm buffer */
- if (g_pFile)
- fwrite(buffer, 1, BUFFER_LENGTH, g_pFile);
#endif
- }
-}
-
static void wakeup_engine_wakeup_event_cb(wakeup_event_info info, void* user_data)
{
MWR_LOGD("[ENTER]");
}
}
-static void join_recorder_thread()
-{
- if (g_recorder_thread.joinable()) {
- MWR_LOGD("g_recorder_thread is joinable, trying join()");
- g_recorder_thread_should_stop.store(true);
- g_recorder_thread.join();
- }
-}
-
-static void start_recorder_thread()
-{
- join_recorder_thread();
-
-#ifdef TV_BT_MODE
- /* Do not start normal recorder thread if TV_BT_MODE and g_voice_key_pressed,
- just send bt_hid start message */
- if (g_voice_key_pressed)
- {
- const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00};
- int bt_retry = 0;
- const int max_retry = 5;
- while (max_retry > bt_retry) {
- int ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data));
- if (BT_ERROR_NONE == ret) {
- MWR_LOGD("[Recorder] Start bt audio recorder");
- break;
- } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
- MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command : %d", ret);
- usleep(50000);
- bt_retry++;
- } else {
- break;
- }
- }
- if (max_retry == bt_retry) {
- MWR_LOGE("[Recorder ERROR] Fail to start bt audio");
- return;
- }
-
- g_bt_extend_count = 0;
- }
- else
-#endif
- {
- g_recorder_thread_should_stop.store(false);
- g_recorder_thread = std::thread(recorder_thread_func);
- }
-}
-
static void wakeup_engine_audio_data_require_status_cb(bool require, void* user_data)
{
MWR_LOGD("[ENTER]");
g_audio_data_required = true;
if (g_audio_data_required != prev_audio_data_required &&
g_voice_key_pressed != true) {
- start_recorder_thread();
+ g_audio_manager.start_recording();
}
} else {
g_audio_data_required = false;
if (g_audio_data_required != prev_audio_data_required &&
g_voice_key_pressed != true) {
- join_recorder_thread();
+ g_audio_manager.stop_recording();
}
}
// UNLOCK REQUIRED
void wakeup_policy_initialize(void)
{
- g_wakeup_policy.reset(new CWakeupPolicyDefault);
+ g_wakeup_policy.reset(new multiassistant::wakeup::CWakeupPolicyDefault);
if (g_wakeup_policy) {
g_wakeup_policy->subscribe(&g_wakeup_event_observer);
}
/* Default Policy specific initialization */
- CWakeupPolicyDefault *default_policy = dynamic_cast<CWakeupPolicyDefault*>(g_wakeup_policy.get());
+ multiassistant::wakeup::CWakeupPolicyDefault *default_policy =
+ dynamic_cast<multiassistant::wakeup::CWakeupPolicyDefault*>(g_wakeup_policy.get());
if (default_policy) {
int priority = 0;
wakeup_policy_initialize();
- recorder_initialize();
+ g_audio_manager.initialize();
+ g_audio_manager.subscribe(&g_audio_data_observer);
+
wakeup_engine_info_initialize();
+#ifdef TV_PRODUCT
+ _grab_voice_key();
+ _add_key_cb();
+#endif
+
MWR_LOGD("[END]");
return 0;
}
{
MWR_LOGD("[ENTER]");
- recorder_deinitialize();
+#ifdef TV_PRODUCT
+ _delete_key_cb();
+ _ungrab_voice_key();
+#endif
- for (const auto &data : g_speech_data) {
- free(data.buffer);
- }
- g_speech_data.clear();
+ g_audio_manager.unsubscribe(&g_audio_data_observer);
+ g_audio_manager.deinitialize();
if (g_current_language) {
free(g_current_language);
if (event == MA_PLUGIN_EVENT_VOICE_KEY_PRESSED) {
if (g_voice_key_pressed != true) {
/* Clear all existing data */
- for (const auto &speech_data : g_speech_data) {
- if (speech_data.buffer) free(speech_data.buffer);
- }
- g_speech_data.clear();
+ g_audio_manager.clear_audio_data();
g_voice_key_pressed = true;
/* (Re)Start recorder thread using bt hid */
- start_recorder_thread();
+ g_audio_manager.start_recording();
for (int loop = 0;loop < g_engine_count;loop++) {
/* Need to find the default assistant, for now assuming 0 is the one */
if(loop == 0) {
}
} else if (event == MA_PLUGIN_EVENT_VOICE_KEY_RELEASED) {
if (g_voice_key_pressed != false) {
- unsigned char final_buffer[2] = {'\0', };
-
g_voice_key_pressed = false;
+ g_audio_manager.finalize_audio_data();
if (g_audio_data_required == true) {
/* Restart recorder thread using standard mic */
- start_recorder_thread();
+ g_audio_manager.start_recording();
} else {
- join_recorder_thread();
- }
- wakeup_engine_speech_data speech_data;
- speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH;
- speech_data.len = sizeof(final_buffer);
- speech_data.buffer = malloc(speech_data.len);
- if (speech_data.buffer) {
- memcpy(speech_data.buffer, final_buffer, speech_data.len);
- g_speech_data.push_back(speech_data);
+ g_audio_manager.stop_recording();
}
}
}
return 0;
}
-void __wakeup_service_streaming_cb(wakeup_service_speech_streaming_event_e event, void* buffer, unsigned int len)
+void __wakeup_service_streaming_cb(wakeup_speech_streaming_event_e event, void* buffer, unsigned int len)
{
if (WAKEUP_SPEECH_STREAMING_EVENT_START == event) {
MWR_LOGD("streaming_cb START");
}
}
-static void manager_data_thread_func(void)
-{
- MWR_LOGD("[ENTER]");
-
- MWR_LOGD("data_count : %zu", g_speech_data.size());
-
- int index = 0;
-
- while (1) {
- int ret = -1;
- int cnt = 0;
-
- /* get feedback data */
- if (index >= g_speech_data.size()) {
- /* empty queue */
- MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
-
- /* waiting */
- while (1) {
- usleep(10000);
- if (index < g_speech_data.size()) {
- MWR_LOGI("[INFO] Resume thread");
- break;
- }
- if (200 < cnt) {
- MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
- __wakeup_service_streaming_cb(
- WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0);
- return;
- }
- cnt++;
- }
- MWR_LOGI("[INFO] Finish to wait for new feedback data come");
-
- /* resume feedback thread */
- continue;
- }
-
- wakeup_engine_speech_data &speech_data = g_speech_data.at(index);
- __wakeup_service_streaming_cb(
- speech_data.event, speech_data.buffer, speech_data.len);
-
- if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
- MWR_LOGI("[INFO] Finish to get and send speech data");
- break;
- }
-
- index++;
- }
-}
-
static void engine_data_thread_func(void)
{
MWR_LOGD("[ENTER]");
MWR_LOGD("data_count : %d", interface->get_utterance_data_count());
- wakeup_engine_speech_data speech_data;
+ wakeup_speech_data speech_data;
int index = 0;
while (1) {
/* waiting */
while (1) {
- usleep(10000);
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
if (index < interface->get_utterance_data_count()) {
MWR_LOGI("[INFO] Resume thread");
break;
}
}
+void start_engine_data_thread()
+{
+ g_engine_data_thread = std::thread(engine_data_thread_func);
+}
+
+void join_engine_data_thread()
+{
+ if (g_engine_data_thread.joinable()) {
+ MWR_LOGD("g_manager_data_thread is joinable, trying join()");
+ g_engine_data_thread_should_stop.store(true);
+ g_engine_data_thread.join();
+ }
+ g_engine_data_thread_should_stop.store(false);
+}
+
int wakeup_manager_start_streaming_utterance_data(void)
{
MWR_LOGD("[ENTER]");
- if (g_speech_data_thread.joinable()) {
- MWR_LOGD("g_speech_data_thread is joinable, trying join()");
- g_speech_data_thread_should_stop.store(true);
- g_speech_data_thread.join();
- }
- g_speech_data_thread_should_stop.store(false);
+ g_audio_manager.stop_streaming();
+ join_engine_data_thread();
+
if (g_voice_key_pressed) {
- g_speech_data_thread = std::thread(manager_data_thread_func);
+ g_audio_manager.start_streaming();
} else {
- g_speech_data_thread = std::thread(engine_data_thread_func);
+ start_engine_data_thread();
}
MWR_LOGD("[END]");
MWR_LOGD("[ENTER]");
if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_UTTERANCE) {
wakeup_manager_change_state(WAKEUP_MANAGER_STATE_PROCESSING);
- if (g_speech_data_thread.joinable()) {
- MWR_LOGD("g_speech_data_thread is joinable, trying join()");
- g_speech_data_thread_should_stop.store(true);
- g_speech_data_thread.join();
- }
+ g_audio_manager.stop_streaming();
+ join_engine_data_thread();
}
MWR_LOGD("[END]");
return 0;
MWR_LOGD("[END]");
return 0;
}
+
+bool CAudioDataObserver::on_recording_audio_data(long time, void* data, int len)
+{
+ if (!g_audio_data_required) return false;
+
+ for (int loop = 0;loop < g_engine_count;loop++) {
+ if (g_wakeup_engine_info[loop].audio_data_require_status &&
+ g_wakeup_engine_info[loop].interface.feed_audio_data) {
+ int ret = g_wakeup_engine_info[loop].interface.feed_audio_data(time, data, len);
+ if (0 == ret) {
+ LOGE("[ERROR] Fail to feed speech data, ret(%d)", ret);
+ }
+ }
+ }
+ return true;
+}
+
+bool CAudioDataObserver::on_streaming_audio_data(
+ wakeup_speech_streaming_event_e event, void* buffer, unsigned int len)
+{
+ __wakeup_service_streaming_cb(event, buffer, len);
+}
+++ /dev/null
-#include "wakeup_manager_audio.h"
-#include "wakeup_manager_main.h"
-
#include <vector>
#include <algorithm>
+namespace multiassistant
+{
+namespace wakeup
+{
+
void CWakeupPolicyImpl::subscribe(IWakeupEventObserver *observer)
{
mObservers.push_back(observer);
{
if (mImpl) mImpl->unsubscribe(observer);
}
+
+} // wakeup
+} // multiassistant
#define LOG_TAG "WakeupPolicyDefault"
#endif
+namespace multiassistant
+{
+namespace wakeup
+{
+
CWakeupPolicyDefault::CWakeupPolicyDefault()
{
}
mTimer = nullptr;
}
}
+
+} // wakeup
+} // multiassistant
}
-static void __audio_streaming_cb(wakeup_service_speech_streaming_event_e event, unsigned char* buffer, int len, void *user_data)
+static void __audio_streaming_cb(wakeup_speech_streaming_event_e event, unsigned char* buffer, int len, void *user_data)
{
if (event == WAKEUP_SPEECH_STREAMING_EVENT_FAIL) {
mas_client_send_recognition_result(0, MA_RECOGNITION_RESULT_EVENT_ERROR);
#endif
}
-static void __speech_status_cb(wakeup_service_speech_status_e status, void *user_data)
+static void __speech_status_cb(wakeup_speech_status_e status, void *user_data)
{
MAS_LOGD( "[SUCCESS] __speech_status_cb is called, status(%d)", status);
}
projects / platform / core / uifw / multi-assistant-service.git / commitdiff