long wakeup_start_time;
long wakeup_end_time;
+ bool wakeup_time_valid;
const void *extra_data;
int extra_data_length;
#include "wakeup_interfaces.h"
#include <atomic>
+#include <list>
#include <thread>
#include <vector>
void start_recording();
void stop_recording();
- void clear_audio_data();
- void finalize_audio_data();
+ void add_speech_data(wakeup_speech_data& data);
+ void clear_speech_data();
+ void finalize_speech_data();
- void start_streaming();
- void stop_streaming();
+ void start_streaming_current_utterance_data(bool from_start_time = false, long start_time = 0);
+ void stop_streaming_current_utterance_data();
+
+ void start_streaming_previous_utterance_data();
+ void stop_streaming_previous_utterance_data();
+
+ void start_streaming_follow_up_data();
+ void stop_streaming_follow_up_data();
void voice_key_pressed_set(bool pressed);
bool voice_key_pressed_get();
-
- void add_custom_speech_data(wakeup_speech_data& data);
private:
void recorder_thread_func(void);
- void streaming_thread_func(void);
+ void streaming_speech_data_thread_func();
+ void streaming_background_data_thread_func(long start_time);
+
+ long get_current_milliseconds_after_epoch();
std::vector<IAudioDataObserver*> mObservers;
std::thread mStreamingThread;
std::atomic_bool mStopStreamingThread{false};
- std::vector<wakeup_speech_data> mSpeechData;
+ static constexpr long mBackgroundRecordingDurationMilliseconds = 10 * 1000;
+ typedef struct {
+ long time;
+ wakeup_speech_data data;
+ } wakeup_speech_data_with_time;
+ std::vector<wakeup_speech_data_with_time> mSpeechData;
+ std::vector<wakeup_speech_data_with_time> mPreviousSpeechData;
+ std::list<wakeup_speech_data_with_time> mBackgroundData;
bool mVoiceKeyPressed{false};
};
long wakeup_start_time;
long wakeup_end_time;
+ bool wakeup_time_valid;
const void *extra_data;
int extra_data_length;
data.buffer = malloc(voice_data->length);
if (data.buffer) {
memcpy(data.buffer, voice_data->audio_buf, voice_data->length);
- manager->add_custom_speech_data(data);
+ manager->add_speech_data(data);
}
} else {
MWR_LOGE("[Recorder ERROR] voice key seems to be already released");
MWR_LOGD("[Recorder] Bluetooth is available");
}
#endif
+ return 0;
}
int CAudioManager::deinitialize(void)
MWR_LOGD("[ENTER]");
for (const auto &data : mSpeechData) {
- free(data.buffer);
+ free(data.data.buffer);
}
mSpeechData.clear();
while (!(mStopRecorderThread.load())) {
unsigned char buffer[BUFFER_LENGTH];
- int ret;
memset(buffer, '\0', BUFFER_LENGTH);
- auto now = std::chrono::system_clock::now();
- auto now_ms = std::chrono::time_point_cast<std::chrono::milliseconds>(now);
- /* number of milliseconds since the epoch of system_clock */
- auto value = now_ms.time_since_epoch();
-
- static long time = 0;
- if (time == value.count()) {
- LOGE("[Recorder WARNING] Time value duplicated : %lu", time);
- }
- time = value.count();
+ long time = get_current_milliseconds_after_epoch();
int read_bytes = audio_in_read(mAudioIn, buffer, BUFFER_LENGTH);
if (0 > read_bytes) {
}
}
+ long delta = mBackgroundRecordingDurationMilliseconds;
+ if (mBackgroundData.size() > 0) {
+ while(mBackgroundData.size() > 0 && mBackgroundData.front().time < time - delta) {
+ const auto &front = mBackgroundData.front();
+ if (front.data.buffer) {
+ free(front.data.buffer);
+ }
+ mBackgroundData.pop_front();
+ if (0 == buffer_count % 100) {
+ LOGD("[Recorder] list pop_front (%zu)", mBackgroundData.size());
+ }
+ }
+ }
+
+ wakeup_speech_data_with_time data;
+ data.data.buffer = malloc(read_bytes);
+ if (data.data.buffer) {
+ data.time = time;
+ data.data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE;
+ data.data.len = read_bytes;
+ memcpy(data.data.buffer, buffer, read_bytes);
+ mBackgroundData.push_back(data);
+ if (0 == buffer_count % 100) {
+ LOGD("[Recorder] list push_back (%zu), %ld %ld %ld", mBackgroundData.size(), mBackgroundData.front().time, time, time - delta);
+ }
+ }
+
// UNLOCK REQUIRED
/* Audio read log */
if (0 == buffer_count % 100) {
}
-void CAudioManager::streaming_thread_func(void)
+void CAudioManager::streaming_speech_data_thread_func()
{
MWR_LOGD("[ENTER]");
- MWR_LOGD("data_count : %zu", mSpeechData.size());
-
int index = 0;
+ MWR_LOGD("data_count : %zu", mSpeechData.size());
while (!(mStopStreamingThread.load())) {
int ret = -1;
continue;
}
- wakeup_speech_data &speech_data = mSpeechData.at(index);
+ wakeup_speech_data& speech_data = mSpeechData.at(index).data;
for (const auto& observer : mObservers) {
if (observer) {
if (!observer->on_streaming_audio_data(
if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
MWR_LOGI("[INFO] Finish to get and send speech data");
+ /* Now move all the speech data to previous speech data for later use */
+ mPreviousSpeechData = std::move(mSpeechData);
break;
}
}
}
-void CAudioManager::clear_audio_data()
+void CAudioManager::streaming_background_data_thread_func(long start_time)
+{
+ MWR_LOGD("[ENTER]");
+
+ auto lead = mBackgroundData.begin();
+ auto iter = lead;
+ while (lead != mBackgroundData.end() && lead->time < start_time) {
+ iter = lead;
+ std::advance(lead, 1);
+ }
+
+ MWR_LOGD("data_count : %zu", mBackgroundData.size());
+
+ while (!(mStopStreamingThread.load())) {
+ int ret = -1;
+ int cnt = 0;
+
+ /* get feedback data */
+ if (lead == mBackgroundData.end()) {
+ /* empty queue */
+ MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
+
+ /* waiting */
+ while (1) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ if (iter == mBackgroundData.end()) {
+ iter = mBackgroundData.begin();
+ }
+ lead = iter;
+ if (lead != mBackgroundData.end()) {
+ std::advance(lead, 1);
+ }
+ if (lead != mBackgroundData.end()) {
+ MWR_LOGI("[INFO] Resume thread");
+ break;
+ }
+ if (200 < cnt) {
+ MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_streaming_audio_data(
+ WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+ return;
+ }
+ cnt++;
+ }
+ MWR_LOGI("[INFO] Finish to wait for new feedback data come");
+
+ /* resume feedback thread */
+ continue;
+ }
+
+ iter = lead;
+ /* Extracted background data will be used as previous utterance data*/
+ mSpeechData.push_back(*iter);
+
+ wakeup_speech_data& speech_data = iter->data;
+ for (const auto& observer : mObservers) {
+ if (observer) {
+ if (!observer->on_streaming_audio_data(
+ speech_data.event, speech_data.buffer, speech_data.len)) {
+ LOGE("[Recorder WARNING] One of the observer returned false");
+ }
+ }
+ }
+
+ if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
+ MWR_LOGI("[INFO] Finish to get and send speech data");
+ /* Now move all the speech data to previous speech data for later use */
+ mPreviousSpeechData = std::move(mSpeechData);
+ break;
+ }
+
+ std::advance(lead, 1);
+ }
+}
+
+void CAudioManager::add_speech_data(wakeup_speech_data& speech_data)
+{
+ wakeup_speech_data_with_time data;
+ data.data = speech_data;
+ data.time = get_current_milliseconds_after_epoch();
+ mSpeechData.push_back(data);
+}
+
+void CAudioManager::clear_speech_data()
{
- for (const auto &speech_data : mSpeechData) {
- if (speech_data.buffer) free(speech_data.buffer);
+ for (const auto &data : mSpeechData) {
+ if (data.data.buffer) free(data.data.buffer);
}
mSpeechData.clear();
}
-void CAudioManager::finalize_audio_data()
+void CAudioManager::finalize_speech_data()
{
unsigned char final_buffer[2] = {'\0', };
wakeup_speech_data speech_data;
speech_data.buffer = malloc(speech_data.len);
if (speech_data.buffer) {
memcpy(speech_data.buffer, final_buffer, speech_data.len);
- mSpeechData.push_back(speech_data);
+ wakeup_speech_data_with_time data;
+ data.data = speech_data;
+ data.time = get_current_milliseconds_after_epoch();
+ mSpeechData.push_back(data);
}
}
-void CAudioManager::start_streaming()
+void CAudioManager::start_streaming_current_utterance_data(bool from_start_time, long start_time)
{
- mStreamingThread = std::thread(&CAudioManager::streaming_thread_func, this);
+ if (from_start_time) {
+ mSpeechData.clear();
+ mStreamingThread = std::thread(&CAudioManager::streaming_background_data_thread_func, this, start_time);
+ } else {
+ mStreamingThread = std::thread(&CAudioManager::streaming_speech_data_thread_func, this);
+ }
}
-void CAudioManager::stop_streaming()
+void CAudioManager::stop_streaming_current_utterance_data()
{
if (mStreamingThread.joinable()) {
MWR_LOGD("mStreamingThread is joinable, trying join()");
mStreamingThread.join();
}
mStopStreamingThread.store(false);
+
+ /* Now move all the speech data to previous speech data for later use */
+ mPreviousSpeechData = std::move(mSpeechData);
+}
+
+void CAudioManager::start_streaming_previous_utterance_data()
+{
+}
+
+void CAudioManager::stop_streaming_previous_utterance_data()
+{
+}
+
+void CAudioManager::start_streaming_follow_up_data()
+{
+}
+
+void CAudioManager::stop_streaming_follow_up_data()
+{
}
void CAudioManager::voice_key_pressed_set(bool pressed)
return mVoiceKeyPressed;
}
-void CAudioManager::add_custom_speech_data(wakeup_speech_data& data)
+long CAudioManager::get_current_milliseconds_after_epoch()
{
- mSpeechData.push_back(data);
+ auto now = std::chrono::system_clock::now();
+ auto now_ms = std::chrono::time_point_cast<std::chrono::milliseconds>(now);
+ /* number of milliseconds since the epoch of system_clock */
+ auto value = now_ms.time_since_epoch();
+
+ return value.count();
}
} // wakeup
static char* g_current_language = NULL;
static wakeup_manager_state_e g_wakeup_manager_state;
+static wakeup_event_info g_last_wakeup_event_info;
+
#define DEFAULT_ASSISTANT_APPID "com.samsung.bixby-voice"
#define WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid"
#define WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS "db/multi-assistant/enabled_assistants"
#define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_DELAY "db/multi-assistant/wakeup_policy_delay"
#define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_PRIORITY "db/multi-assistant/wakeup_policy_priority"
+#define WAKEUP_SETTINGS_KEY_STREAMING_DURATION_MAX "db/multi-assistant/streaming_duration_max"
typedef struct {
std::string default_assistant_appid{DEFAULT_ASSISTANT_APPID};
std::vector<std::string> enabled_assistants{DEFAULT_ASSISTANT_APPID};
float wakeup_policy_delay{0.1};
std::vector<std::string> wakeup_policy_priority; // No priority by default
+ float streaming_duration_max{10.0};
} wakeup_settings;
static wakeup_settings g_wakeup_settings;
+enum STREAMING_MODE {
+ STREAMING_MODE_NONE,
+ STREAMING_MODE_UTTERANCE,
+ STREAMING_MODE_PREVIOUS_UTTERANCE,
+ STREAMING_MODE_FOLLOW_UP,
+};
+
+static STREAMING_MODE g_streaming_mode{STREAMING_MODE_NONE};
+static Ecore_Timer* g_streaming_duration_timer;
+
class CWakeupEventObserver : public multiassistant::wakeup::IWakeupEventObserver
{
public:
/* We'll need to check vconf for enabled wakeup engines */
g_wakeup_engine_info[loop].enabled = true;
g_wakeup_engine_info[loop].audio_data_require_status = false;
+
MWR_LOGD("Initializing wakeup engine : %s %p", g_wakeup_engine_info[loop].engine_path, g_wakeup_engine_info[loop].interface.initialize);
if (g_wakeup_engine_info[loop].interface.initialize) {
g_wakeup_engine_info[loop].interface.initialize();
void CWakeupEventObserver::on_wakeup(wakeup_event_info info)
{
if (NULL != g_wakeup_event_cb) {
+ g_last_wakeup_event_info = info;
g_wakeup_event_cb(info, g_wakeup_event_user_data);
}
}
free(vconf_str);
vconf_str = nullptr;
}
+ vconf_ret = vconf_get_dbl(WAKEUP_SETTINGS_KEY_STREAMING_DURATION_MAX, &vconf_double);
+ if (0 == vconf_ret) {
+ g_wakeup_settings.streaming_duration_max = vconf_double;
+ MWR_LOGD("streaming_duration_max : %f", g_wakeup_settings.streaming_duration_max);
+ }
wakeup_policy_initialize();
g_wakeup_engine_info[loop].interface.update_manager_state(state);
}
}
+ return 0;
}
int wakeup_manager_activate(void)
if (event == MA_PLUGIN_EVENT_VOICE_KEY_PRESSED) {
if (g_voice_key_pressed != true) {
/* Clear all existing data */
- g_audio_manager.clear_audio_data();
+ g_audio_manager.clear_speech_data();
g_voice_key_pressed = true;
/* (Re)Start recorder thread using bt hid */
} else if (event == MA_PLUGIN_EVENT_VOICE_KEY_RELEASED) {
if (g_voice_key_pressed != false) {
g_voice_key_pressed = false;
- g_audio_manager.finalize_audio_data();
+ g_audio_manager.finalize_speech_data();
if (g_audio_data_required == true) {
/* Restart recorder thread using standard mic */
g_audio_manager.start_recording();
g_engine_data_thread_should_stop.store(false);
}
+static Eina_Bool streaming_duration_expired(void *data)
+{
+ MWR_LOGD("[ENTER]");
+ switch(g_streaming_mode) {
+ case STREAMING_MODE_UTTERANCE:
+ g_audio_manager.stop_streaming_current_utterance_data();
+ join_engine_data_thread();
+ break;
+ case STREAMING_MODE_PREVIOUS_UTTERANCE:
+ g_audio_manager.stop_streaming_previous_utterance_data();
+ break;
+ case STREAMING_MODE_FOLLOW_UP:
+ g_audio_manager.stop_streaming_follow_up_data();
+ break;
+ }
+
+ unsigned char final_buffer[2] = {'\0', };
+ __wakeup_service_streaming_cb(
+ WAKEUP_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer));
+
+ g_streaming_mode = STREAMING_MODE_NONE;
+ return ECORE_CALLBACK_CANCEL;
+}
+
int wakeup_manager_start_streaming_utterance_data(void)
{
MWR_LOGD("[ENTER]");
- g_audio_manager.stop_streaming();
+ g_streaming_mode = STREAMING_MODE_UTTERANCE;
+
+ g_audio_manager.stop_streaming_current_utterance_data();
join_engine_data_thread();
+ /* What if the user pressed voice key but then again immediately releases? */
if (g_voice_key_pressed) {
- g_audio_manager.start_streaming();
+ g_audio_manager.start_streaming_current_utterance_data();
} else {
- start_engine_data_thread();
+ if(g_last_wakeup_event_info.wakeup_time_valid) {
+ g_audio_manager.start_streaming_current_utterance_data(true, g_last_wakeup_event_info.wakeup_end_time);
+ } else {
+ start_engine_data_thread();
+ }
}
+ ecore_thread_main_loop_begin();
+ if (g_streaming_duration_timer) {
+ ecore_timer_del(g_streaming_duration_timer);
+ }
+ g_streaming_duration_timer = ecore_timer_add(g_wakeup_settings.streaming_duration_max,
+ streaming_duration_expired, nullptr);
+ ecore_thread_main_loop_end();
+
MWR_LOGD("[END]");
return 0;
}
int wakeup_manager_stop_streaming_utterance_data(void)
{
MWR_LOGD("[ENTER]");
+ if (g_streaming_duration_timer) {
+ ecore_thread_main_loop_begin();
+ ecore_timer_del(g_streaming_duration_timer);
+ ecore_thread_main_loop_end();
+ }
if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_UTTERANCE) {
wakeup_manager_change_state(WAKEUP_MANAGER_STATE_PROCESSING);
- g_audio_manager.stop_streaming();
- join_engine_data_thread();
}
+ g_audio_manager.stop_streaming_current_utterance_data();
+ join_engine_data_thread();
+ g_streaming_mode = STREAMING_MODE_NONE;
MWR_LOGD("[END]");
return 0;
}
{
MWR_LOGD("[ENTER]");
+ g_streaming_mode = STREAMING_MODE_FOLLOW_UP;
+
MWR_LOGD("[END]");
return 0;
}
{
MWR_LOGD("[ENTER]");
+ g_streaming_mode = STREAMING_MODE_NONE;
+
MWR_LOGD("[END]");
return 0;
}
{
MWR_LOGD("[ENTER]");
+ g_streaming_mode = STREAMING_MODE_PREVIOUS_UTTERANCE;
+
MWR_LOGD("[END]");
return 0;
}
{
MWR_LOGD("[ENTER]");
+ g_streaming_mode = STREAMING_MODE_NONE;
+
MWR_LOGD("[END]");
return 0;
}
wakeup_speech_streaming_event_e event, void* buffer, unsigned int len)
{
__wakeup_service_streaming_cb(event, buffer, len);
+ if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == event) {
+ g_streaming_mode = STREAMING_MODE_NONE;
+ }
+ return true;
}
{
int ret = -1;
if (NULL != g_handle) {
- wakeup_manager_send_assistant_specific_command func = _wakeup_manager_interface.send_assistant_specific_command;
+ wakeup_manager_send_assistant_specific_command func = _wakeup_manager_interface.send_assistant_specific_command;
if (NULL == func) {
MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SEND_ASSISTANT_SPECIFIC_COMMAND);
} else {
projects / platform / core / uifw / multi-assistant-service.git / commitdiff