plugins/wakeup-manager/src/wakeup_audio_manager.cpp

   1 #include "wakeup_audio_manager.h"
   2 #include "wakeup_manager_main.h"
   3 #include "dependency_resolver.h"
   4 #include "heap_tracer.h"
   5
   6 #include <algorithm>
   7
   8 #include <Ecore.h>
   9 #include <audio_io.h>
  10 #include <sound_manager.h>
  11 #include <sound_manager_internal.h>
  12
  13 namespace multiassistant
  14 {
  15 namespace wakeup
  16 {
  17
  18 /* Need to check whether this value needs to be configurable */
  19 static int g_speech_pcm_wait_count = 800;
  20
  21 static long long get_current_milliseconds_after_epoch()
  22 {
  23         auto now = chrono::steady_clock::now();
  24         auto now_ms = chrono::time_point_cast<chrono::milliseconds>(now);
  25         /* number of milliseconds since the epoch of system_clock */
  26         auto value = now_ms.time_since_epoch();
  27
  28         return value.count();
  29 }
  30
  31 CAudioManager::CAudioManager()
  32 {
  33 }
  34
  35 CAudioManager::~CAudioManager()
  36 {
  37 }
  38
  39 CAudioManager::CAudioManager(IAudioEventObserver *observer) : CAudioManager()
  40 {
  41         subscribe(observer);
  42 }
  43
  44 void recording_focus_state_watch_cb(int id, sound_stream_focus_mask_e focus_mask,
  45         sound_stream_focus_state_e focus_state, sound_stream_focus_change_reason_e reason,
  46         const char *extra_info, void *user_data)
  47 {
  48         MWR_LOGD("[Recorder] focus_mask : %d, focus_state : %d, reason : %d, [%s]",
  49                 focus_mask, focus_state, reason, extra_info);
  50
  51         if (nullptr == user_data) return;
  52
  53         /* The API description states that calling sound_manager_get_current_recording_focus()
  54                 function inside sound_stream_focus_state_watch_cb() function is prohibited */
  55         ecore_main_loop_thread_safe_call_async(
  56                 [](void* data) {
  57                         CAudioManager *manager = static_cast<CAudioManager*>(data);
  58                         if (manager) {
  59                                 manager->sound_focus_changed();
  60                         }
  61                 }, user_data);
  62 }
  63
  64 int CAudioManager::initialize(void)
  65 {
  66         sound_manager_add_focus_state_watch_cb(SOUND_STREAM_FOCUS_FOR_RECORDING,
  67                 recording_focus_state_watch_cb, this, &mSoundFocusWatchId);
  68         return 0;
  69 }
  70
  71 int CAudioManager::deinitialize(void)
  72 {
  73         clear_audio_data();
  74         if (mStreamingThread.joinable()) {
  75                 MWR_LOGD("mStreamingThread is joinable, trying join()");
  76                 mStopStreamingThread.store(true);
  77                 try {
  78                         mStreamingThread.join();
  79                 } catch (std::exception &e) {
  80                         MWR_LOGE("Exception thrown : %s", e.what());
  81                 }
  82         }
  83         mStopStreamingThread.store(false);
  84
  85         sound_manager_remove_focus_state_watch_cb(mSoundFocusWatchId);
  86
  87         return 0;
  88 }
  89
  90 void CAudioManager::sound_focus_changed()
  91 {
  92         sound_stream_focus_change_reason_e acquired_by;
  93         int sound_behavior;
  94         char* extra_info = NULL;
  95         int focus = sound_manager_get_current_recording_focus(&acquired_by, &sound_behavior, &extra_info);
  96         MWR_LOGD("[Recorder] sound focus has changed : %d %d %d %s", focus,
  97                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? acquired_by : -1),
  98                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? sound_behavior : -1),
  99                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? extra_info : ""));
 100         if (SOUND_MANAGER_ERROR_NO_DATA == focus) {
 101                 if (mRecordingRequired && !mIsRecording) {
 102                         MWR_LOGD("[Recorder] Currently no other process has acquired sound focus, start recording");
 103                         start_recording(false);
 104                 }
 105         } else {
 106                 if (mIsRecording) {
 107                         MWR_LOGW("[Recorder] Sound focus acquired by other process, stop recording");
 108                         stop_recording(false);
 109                 }
 110         }
 111         if (extra_info) {
 112                 free(extra_info);
 113                 extra_info = NULL;
 114         }
 115 }
 116
 117 void CAudioManager::subscribe(IAudioEventObserver *observer)
 118 {
 119         if (observer) {
 120                 mObservers.push_back(observer);
 121         }
 122 }
 123
 124 void CAudioManager::unsubscribe(IAudioEventObserver *observer)
 125 {
 126         auto iter = find(mObservers.begin(), mObservers.end(), observer);
 127         if (iter != mObservers.end()) {
 128                 mObservers.erase(iter);
 129         }
 130 }
 131
 132 void CAudioManager::stop_recording(bool proactive)
 133 {
 134         dependency_resolver_stop_recording();
 135         if (proactive) {
 136                 mRecordingRequired = false;
 137         }
 138         mIsRecording = false;
 139 }
 140
 141 void CAudioManager::start_recording(bool proactive)
 142 {
 143         if (mIsRecording) {
 144                 stop_recording(false);
 145         }
 146         if (proactive) {
 147                 mRecordingRequired = true;
 148         }
 149
 150         sound_stream_focus_change_reason_e acquired_by;
 151         int sound_behavior;
 152         char* extra_info = NULL;
 153         if (SOUND_MANAGER_ERROR_NO_DATA == sound_manager_get_current_recording_focus(&acquired_by, &sound_behavior, &extra_info)) {
 154                 MWR_LOGD("[Recorder] Currently no other process has acquired sound focus, start recording");
 155                 dependency_resolver_start_recording();
 156                 mIsRecording = true;
 157         } else {
 158                 MWR_LOGW("[Recorder] Currently sound focus is acquired by other process, skip recording");
 159         }
 160         if (extra_info) {
 161                 free(extra_info);
 162                 extra_info = NULL;
 163         }
 164 }
 165
 166 void CAudioManager::set_recording_session(recording_session session)
 167 {
 168         dependency_resolver_set_recording_session((unsigned int)session);
 169 }
 170
 171 /* Need to consider adapting conventional producer-consumer model */
 172 void CAudioManager::streaming_previous_audio_data_thread_func()
 173 {
 174         MWR_LOGI("[ENTER]");
 175
 176         unique_lock<mutex> lock(mMutex, defer_lock);
 177
 178         /* get feedback data */
 179         size_t audio_data_size = 0;
 180         lock.lock();
 181         audio_data_size = mPreviousAudioData.size();
 182         for (int index = 0; index < audio_data_size; index++) {
 183                 mas_speech_data& speech_data = mPreviousAudioData.at(index).data;
 184                 for (const auto& observer : mObservers) {
 185                         if (observer) {
 186                                 if (!observer->on_streaming_audio_data(
 187                                         speech_data.event, speech_data.buffer, speech_data.len)) {
 188                                         LOGE("[Recorder WARNING] One of the observer returned false");
 189                                 }
 190                         }
 191
 192                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 193                                 MWR_LOGI("[INFO] Finish to send previous speech data");
 194                                 return;
 195                         }
 196                 }
 197
 198                 if (mStopStreamingThread.load()) {
 199                         MWR_LOGI("[INFO] Stop Streaming Requested, returning");
 200                         return;
 201                 }
 202         }
 203         lock.unlock();
 204
 205         MWR_LOGI("[EXIT]");
 206 }
 207
 208 void CAudioManager::streaming_audio_data_thread_func(long long start_time)
 209 {
 210         MWR_LOGI("[ENTER]");
 211
 212         unique_lock<mutex> lock(mMutex, defer_lock);
 213         bool finish_event_sent = false;
 214
 215         lock.lock();
 216         auto lead = mAudioData.begin();
 217         auto iter = lead;
 218         while (lead != mAudioData.end() && lead->time < start_time) {
 219                 iter = lead;
 220                 advance(lead, 1);
 221         }
 222         MWR_LOGI("data_count : %zu", mAudioData.size());
 223         lock.unlock();
 224
 225         while (!(mStopStreamingThread.load())) {
 226                 int ret = -1;
 227                 int cnt = 0;
 228
 229                 /* get feedback data */
 230                 lock.lock();
 231                 auto end = mAudioData.end();
 232                 lock.unlock();
 233                 if (lead == end) {
 234                         /* empty queue */
 235                         MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
 236
 237                         /* waiting */
 238                         while (!(mStopStreamingThread.load())) {
 239                                 this_thread::sleep_for(chrono::milliseconds(10));
 240                                 lock.lock();
 241                                 end = mAudioData.end();
 242                                 auto begin = mAudioData.begin();
 243                                 lock.unlock();
 244                                 if (iter == end) {
 245                                         lead = begin;
 246                                 } else {
 247                                         lead = iter;
 248                                         if (lead != end) {
 249                                                 advance(lead, 1);
 250                                         }
 251                                 }
 252                                 if (lead != end) {
 253                                         MWR_LOGD("[INFO] Resume thread");
 254                                         break;
 255                                 }
 256                                 if (g_speech_pcm_wait_count < cnt) {
 257                                         unsigned char final_buffer[2] = {'\0', };
 258                                         MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
 259                                         for (const auto& observer : mObservers) {
 260                                                 if (observer) {
 261                                                         if (!observer->on_streaming_audio_data(
 262                                                                 MAS_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
 263                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 264                                                         }
 265                                                         if (!observer->on_streaming_audio_data(
 266                                                                 MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 267                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 268                                                         }
 269                                                 }
 270                                         }
 271                                         return;
 272                                 }
 273                                 cnt++;
 274                         }
 275                         MWR_LOGD("[INFO] Finish to wait for new feedback data come");
 276
 277                         /* resume feedback thread */
 278                         continue;
 279                 }
 280
 281                 /* FIXME : Extracted audio data here should be used as previous audio data*/
 282
 283                 lock.lock();
 284                 end = mAudioData.end();
 285
 286                 if (lead != end) {
 287                         iter = lead;
 288                         mas_speech_data& speech_data = iter->data;
 289                         for (const auto& observer : mObservers) {
 290                                 if (observer) {
 291                                         if (!observer->on_streaming_audio_data(
 292                                                 speech_data.event, speech_data.buffer, speech_data.len)) {
 293                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 294                                         }
 295                                 }
 296                         }
 297
 298                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 299                                 MWR_LOGI("[INFO] Finish to get and send speech data");
 300                                 finish_event_sent = true;
 301                                 break;
 302                         }
 303
 304                         advance(lead, 1);
 305                 }
 306                 lock.unlock();
 307         }
 308
 309         if (true != finish_event_sent) {
 310                 unsigned char final_buffer[2] = {'\0', };
 311                 for (const auto& observer : mObservers) {
 312                         if (observer) {
 313                                 MWR_LOGI("No FINISH event sent yet, adding to finalize streaming session");
 314                                 if (!observer->on_streaming_audio_data(
 315                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 316                                         LOGE("[Recorder WARNING] One of the observer returned false");
 317                                 }
 318                         }
 319                 }
 320         }
 321         MWR_LOGI("[EXIT]");
 322 }
 323
 324 void CAudioManager::add_audio_data(mas_speech_data& data, long long time)
 325 {
 326         long long delta = mAudioRecordingDurationMilliseconds;
 327
 328         notify_audio_data_recording(time, data.buffer, data.len);
 329
 330         mas_speech_data_with_time data_with_time;
 331         data_with_time.data = data;
 332         data_with_time.time = time;
 333
 334         lock_guard<mutex> lock(mMutex);
 335
 336         /* Pop items only when the streaming is not activated */
 337         while(false == mAudioData.empty() && mAudioData.front().time < time - delta) {
 338                 const auto &front = mAudioData.front();
 339                 if (front.data.buffer) {
 340                         vm_free_simple(front.data.buffer);
 341                 }
 342                 mAudioData.pop_front();
 343         }
 344         mAudioData.push_back(data_with_time);
 345 }
 346
 347 void CAudioManager::feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len)
 348 {
 349         if (NULL == buffer || 0 == len) return;
 350
 351         mas_speech_data speech_data;
 352         speech_data.buffer = vm_malloc_simple(len);
 353         if (speech_data.buffer) {
 354                 long long time = get_current_milliseconds_after_epoch();
 355
 356                 speech_data.event = event;
 357                 speech_data.len = len;
 358                 memcpy(speech_data.buffer, buffer, len);
 359                 add_audio_data(speech_data, time);
 360         }
 361 }
 362
 363 void CAudioManager::finalize_audio_data()
 364 {
 365         MWR_LOGI("Adding FINISH event to audio data list");
 366         unsigned char final_buffer[2] = {'\0', };
 367         mas_speech_data speech_data;
 368         speech_data.event = MAS_SPEECH_STREAMING_EVENT_FINISH;
 369         speech_data.len = sizeof(final_buffer);
 370         speech_data.buffer = vm_malloc_simple(speech_data.len);
 371         if (speech_data.buffer) {
 372                 long long time = get_current_milliseconds_after_epoch();
 373
 374                 memcpy(speech_data.buffer, final_buffer, speech_data.len);
 375                 add_audio_data(speech_data, time);
 376         }
 377 }
 378
 379 void CAudioManager::clear_audio_data()
 380 {
 381         lock_guard<mutex> lock(mMutex);
 382         while(!mAudioData.empty()) {
 383                 const auto &front = mAudioData.front();
 384                 if (front.data.buffer) {
 385                         vm_free_simple(front.data.buffer);
 386                 }
 387                 mAudioData.pop_front();
 388         }
 389         mAudioData.clear();
 390 }
 391
 392 void CAudioManager::notify_audio_data_recording(long time, void* data, int len)
 393 {
 394         for (const auto& observer : mObservers) {
 395                 if (observer) {
 396                         if (!observer->on_recording_audio_data(time, data, len)) {
 397                                 LOGE("[Recorder WARNING] One of the observer returned false");
 398                         }
 399                 }
 400         }
 401 }
 402
 403 void CAudioManager::start_streaming_current_utterance_data(long long start_time)
 404 {
 405         MWR_LOGI("[ENTER]");
 406         if (mStreamingThread.joinable()) {
 407                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 408                 return;
 409         }
 410         lock_guard<mutex> lock(mMutex);
 411         mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, start_time);
 412 }
 413
 414 void CAudioManager::stop_streaming_current_utterance_data()
 415 {
 416         MWR_LOGI("[ENTER]");
 417         if (mStreamingThread.joinable()) {
 418                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 419                 mStopStreamingThread.store(true);
 420                 try {
 421                         mStreamingThread.join();
 422                 } catch (std::exception &e) {
 423                         MWR_LOGE("Exception thrown : %s", e.what());
 424                 }
 425         }
 426         mStopStreamingThread.store(false);
 427
 428         /* FIXME : Need to move all the speech data to previous speech data for later use */
 429 }
 430
 431 void CAudioManager::start_streaming_previous_utterance_data()
 432 {
 433         MWR_LOGI("[ENTER]");
 434         if (mStreamingPreviousThread.joinable()) {
 435                 MWR_LOGE("ERROR : mStreamingPreviousThread is joinable, will not start a new thread");
 436                 return;
 437         }
 438         lock_guard<mutex> lock(mMutex);
 439         mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_audio_data_thread_func, this);
 440 }
 441
 442 void CAudioManager::stop_streaming_previous_utterance_data()
 443 {
 444         MWR_LOGI("[ENTER]");
 445         if (mStreamingPreviousThread.joinable()) {
 446                 MWR_LOGD("mStreamingPreviousThread is joinable, trying join()");
 447                 mStopStreamingPreviousThread.store(true);
 448                 try {
 449                         mStreamingPreviousThread.join();
 450                 } catch (std::exception &e) {
 451                         MWR_LOGE("Exception thrown : %s", e.what());
 452                 }
 453         }
 454         mStopStreamingThread.store(false);
 455 }
 456
 457 void CAudioManager::start_streaming_follow_up_data()
 458 {
 459         MWR_LOGI("[ENTER]");
 460         if (mStreamingThread.joinable()) {
 461                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 462                 return;
 463         }
 464
 465         mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, 0);
 466 }
 467
 468 void CAudioManager::stop_streaming_follow_up_data()
 469 {
 470         MWR_LOGI("[ENTER]");
 471         if (mStreamingThread.joinable()) {
 472                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 473                 mStopStreamingThread.store(true);
 474                 mStreamingThread.join();
 475         }
 476         mStopStreamingThread.store(false);
 477 }
 478
 479 void CAudioManager::set_background_volume(double ratio)
 480 {
 481         dependency_resolver_set_background_volume(ratio);
 482 }
 483
 484 } // wakeup
 485 } // multiassistant