plugins/wakeup-manager/src/wakeup_audio_manager.cpp

   1 #include "wakeup_audio_manager.h"
   2 #include "wakeup_manager_main.h"
   3 #include "dependency_resolver.h"
   4 #include "heap_tracer.h"
   5
   6 #include <algorithm>
   7
   8 #include <Ecore.h>
   9 #include <audio_io.h>
  10 #include <sound_manager.h>
  11 #include <sound_manager_internal.h>
  12
  13 namespace multiassistant
  14 {
  15 namespace wakeup
  16 {
  17
  18 /* Need to check whether this value needs to be configurable */
  19 static int g_speech_pcm_wait_count = 800;
  20
  21 static long long get_current_milliseconds_after_epoch()
  22 {
  23         auto now = chrono::steady_clock::now();
  24         auto now_ms = chrono::time_point_cast<chrono::milliseconds>(now);
  25         /* number of milliseconds since the epoch of system_clock */
  26         auto value = now_ms.time_since_epoch();
  27
  28         return value.count();
  29 }
  30
  31 CAudioManager::CAudioManager()
  32 {
  33 }
  34
  35 CAudioManager::~CAudioManager()
  36 {
  37         deinitialize();
  38 }
  39
  40 CAudioManager::CAudioManager(IAudioEventObserver *observer) : CAudioManager()
  41 {
  42         subscribe(observer);
  43 }
  44
  45 void recording_focus_state_watch_cb(int id, sound_stream_focus_mask_e focus_mask,
  46         sound_stream_focus_state_e focus_state, sound_stream_focus_change_reason_e reason,
  47         const char *extra_info, void *user_data)
  48 {
  49         MWR_LOGD("[Recorder] focus_mask : %d, focus_state : %d, reason : %d, [%s]",
  50                 focus_mask, focus_state, reason, extra_info);
  51
  52         if (nullptr == user_data) return;
  53
  54         /* The API description states that calling sound_manager_get_current_recording_focus()
  55                 function inside sound_stream_focus_state_watch_cb() function is prohibited */
  56         ecore_main_loop_thread_safe_call_async(
  57                 [](void* data) {
  58                         CAudioManager *manager = static_cast<CAudioManager*>(data);
  59                         if (manager) {
  60                                 manager->sound_focus_changed();
  61                         }
  62                 }, user_data);
  63 }
  64
  65 int CAudioManager::initialize(void)
  66 {
  67         sound_manager_add_focus_state_watch_cb(SOUND_STREAM_FOCUS_FOR_RECORDING,
  68                 recording_focus_state_watch_cb, this, &mSoundFocusWatchId);
  69         return 0;
  70 }
  71
  72 int CAudioManager::deinitialize(void)
  73 {
  74         clear_audio_data();
  75         if (mStreamingThread.joinable()) {
  76                 MWR_LOGD("mStreamingThread is joinable, trying join()");
  77                 mStopStreamingThread.store(true);
  78                 try {
  79                         mStreamingThread.join();
  80                 } catch (std::exception &e) {
  81                         MWR_LOGE("Exception thrown : %s", e.what());
  82                 }
  83         }
  84         mStopStreamingThread.store(false);
  85
  86         sound_manager_remove_focus_state_watch_cb(mSoundFocusWatchId);
  87
  88         return 0;
  89 }
  90
  91 void CAudioManager::sound_focus_changed()
  92 {
  93         sound_stream_focus_change_reason_e acquired_by;
  94         int sound_behavior;
  95         char* extra_info = NULL;
  96         int focus = sound_manager_get_current_recording_focus(&acquired_by, &sound_behavior, &extra_info);
  97         MWR_LOGD("[Recorder] sound focus has changed : %d %d %d %s", focus,
  98                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? acquired_by : -1),
  99                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? sound_behavior : -1),
 100                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? extra_info : ""));
 101         if (SOUND_MANAGER_ERROR_NO_DATA == focus) {
 102                 if (mRecordingRequired && !mIsRecording) {
 103                         MWR_LOGD("[Recorder] Currently no other process has acquired sound focus, start recording");
 104                         start_recording(false);
 105                 }
 106         } else {
 107                 if (mIsRecording) {
 108                         MWR_LOGW("[Recorder] Sound focus acquired by other process, stop recording");
 109                         stop_recording(false);
 110                 }
 111         }
 112         if (extra_info) {
 113                 free(extra_info);
 114                 extra_info = NULL;
 115         }
 116 }
 117
 118 void CAudioManager::subscribe(IAudioEventObserver *observer)
 119 {
 120         lock_guard<mutex> lock(mMutex);
 121         if (observer) {
 122                 mObservers.push_back(observer);
 123         }
 124 }
 125
 126 void CAudioManager::unsubscribe(IAudioEventObserver *observer)
 127 {
 128         lock_guard<mutex> lock(mMutex);
 129         auto iter = find(mObservers.begin(), mObservers.end(), observer);
 130         if (iter != mObservers.end()) {
 131                 mObservers.erase(iter);
 132         }
 133 }
 134
 135 void CAudioManager::stop_recording(bool proactive)
 136 {
 137         dependency_resolver_stop_recording();
 138         if (proactive) {
 139                 mRecordingRequired = false;
 140         }
 141         mIsRecording = false;
 142 }
 143
 144 void CAudioManager::start_recording(bool proactive)
 145 {
 146         if (mIsRecording) {
 147                 stop_recording(false);
 148         }
 149         if (proactive) {
 150                 mRecordingRequired = true;
 151         }
 152
 153         sound_stream_focus_change_reason_e acquired_by;
 154         int sound_behavior;
 155         char* extra_info = NULL;
 156         if (SOUND_MANAGER_ERROR_NO_DATA == sound_manager_get_current_recording_focus(&acquired_by, &sound_behavior, &extra_info)) {
 157                 MWR_LOGD("[Recorder] Currently no other process has acquired sound focus, start recording");
 158                 dependency_resolver_start_recording();
 159                 mIsRecording = true;
 160         } else {
 161                 MWR_LOGW("[Recorder] Currently sound focus is acquired by other process, skip recording");
 162         }
 163         if (extra_info) {
 164                 free(extra_info);
 165                 extra_info = NULL;
 166         }
 167 }
 168
 169 void CAudioManager::set_recording_session(recording_session session)
 170 {
 171         dependency_resolver_set_recording_session((unsigned int)session);
 172 }
 173
 174 /* Need to consider adapting conventional producer-consumer model */
 175 void CAudioManager::streaming_previous_audio_data_thread_func()
 176 {
 177         MWR_LOGI("[ENTER]");
 178
 179         unique_lock<mutex> lock(mMutex, defer_lock);
 180
 181         /* get feedback data */
 182         size_t audio_data_size = 0;
 183         lock.lock();
 184         audio_data_size = mPreviousAudioData.size();
 185         for (int index = 0; index < audio_data_size; index++) {
 186                 mas_speech_data& speech_data = mPreviousAudioData.at(index).data;
 187                 for (const auto& observer : mObservers) {
 188                         if (observer) {
 189                                 if (!observer->on_streaming_audio_data(
 190                                         speech_data.event, speech_data.buffer, speech_data.len)) {
 191                                         LOGE("[Recorder WARNING] One of the observer returned false");
 192                                 }
 193                         }
 194
 195                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 196                                 MWR_LOGI("[INFO] Finish to send previous speech data");
 197                                 return;
 198                         }
 199                 }
 200
 201                 if (mStopStreamingThread.load()) {
 202                         MWR_LOGI("[INFO] Stop Streaming Requested, returning");
 203                         return;
 204                 }
 205         }
 206         lock.unlock();
 207
 208         MWR_LOGI("[EXIT]");
 209 }
 210
 211 static void validate_audio_data_event_field(const mas_speech_data &data)
 212 {
 213         if (data.event == MAS_SPEECH_STREAMING_EVENT_CONTINUE ||
 214                 data.event == MAS_SPEECH_STREAMING_EVENT_START ||
 215                 data.event == MAS_SPEECH_STREAMING_EVENT_FINISH ||
 216                 data.event == MAS_SPEECH_STREAMING_EVENT_FAIL) {
 217                 return;
 218         }
 219         MWR_LOGE("mas_speech_data has event field : %d", data.event);
 220 }
 221
 222 void CAudioManager::streaming_audio_data_thread_func(long long start_time)
 223 {
 224         MWR_LOGI("[ENTER]");
 225
 226         mStreamingThreadActive.store(true);
 227
 228         unique_lock<mutex> lock(mMutex, defer_lock);
 229         bool finish_event_sent = false;
 230
 231         lock.lock();
 232         auto lead = mAudioData.begin();
 233         auto iter = lead;
 234         while (lead != mAudioData.end() && lead->time < start_time) {
 235                 iter = lead;
 236                 advance(lead, 1);
 237         }
 238         MWR_LOGE("data_count : %zu", mAudioData.size());
 239         lock.unlock();
 240
 241         while (!(mStopStreamingThread.load())) {
 242                 int ret = -1;
 243                 int cnt = 0;
 244
 245                 /* get feedback data */
 246                 lock.lock();
 247                 auto end = mAudioData.end();
 248                 lock.unlock();
 249                 if (lead == end) {
 250                         /* empty queue */
 251                         MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
 252
 253                         /* waiting */
 254                         while (!(mStopStreamingThread.load())) {
 255                                 this_thread::sleep_for(chrono::milliseconds(10));
 256                                 lock.lock();
 257                                 end = mAudioData.end();
 258                                 auto begin = mAudioData.begin();
 259                                 vector<IAudioEventObserver*> observers = mObservers;
 260                                 lock.unlock();
 261                                 if (iter == end) {
 262                                         lead = begin;
 263                                 } else {
 264                                         lead = iter;
 265                                         if (lead != end) {
 266                                                 advance(lead, 1);
 267                                         }
 268                                 }
 269                                 if (lead != end) {
 270                                         MWR_LOGD("[INFO] Resume thread");
 271                                         break;
 272                                 }
 273                                 if (g_speech_pcm_wait_count < cnt) {
 274                                         unsigned char final_buffer[2] = {'\0', };
 275                                         MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
 276                                         for (const auto& observer : observers) {
 277                                                 if (observer) {
 278                                                         if (!observer->on_streaming_audio_data(
 279                                                                 MAS_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
 280                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 281                                                         }
 282                                                         if (!observer->on_streaming_audio_data(
 283                                                                 MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 284                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 285                                                         }
 286                                                 }
 287                                         }
 288                                         mStreamingThreadActive.store(false);
 289                                         return;
 290                                 }
 291                                 cnt++;
 292                         }
 293                         MWR_LOGD("[INFO] Finish to wait for new feedback data come");
 294
 295                         /* resume feedback thread */
 296                         continue;
 297                 }
 298
 299                 /* FIXME : Extracted audio data here should be used as previous audio data*/
 300
 301                 lock.lock();
 302                 end = mAudioData.end();
 303                 vector<IAudioEventObserver*> observers = mObservers;
 304
 305                 if (lead != end) {
 306                         iter = lead;
 307                         mas_speech_data& speech_data = iter->data;
 308                         for (const auto& observer : observers) {
 309                                 if (observer) {
 310                                         validate_audio_data_event_field(speech_data);
 311                                         if (!observer->on_streaming_audio_data(
 312                                                 speech_data.event, speech_data.buffer, speech_data.len)) {
 313                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 314                                         }
 315                                 }
 316                         }
 317
 318                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 319                                 MWR_LOGE("[INFO] Finish to get and send speech data");
 320                                 finish_event_sent = true;
 321                                 break;
 322                         }
 323
 324                         advance(lead, 1);
 325                 }
 326                 lock.unlock();
 327         }
 328
 329         if (true != finish_event_sent) {
 330                 unsigned char final_buffer[2] = {'\0', };
 331                 lock.lock();
 332                 vector<IAudioEventObserver*> observers = mObservers;
 333                 lock.unlock();
 334                 for (const auto& observer : observers) {
 335                         if (observer) {
 336                                 MWR_LOGE("No FINISH event sent yet, adding to finalize streaming session");
 337                                 if (!observer->on_streaming_audio_data(
 338                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 339                                         LOGE("[Recorder WARNING] One of the observer returned false");
 340                                 }
 341                         }
 342                 }
 343         }
 344         mStreamingThreadActive.store(false);
 345         MWR_LOGE("[EXIT]");
 346 }
 347
 348 void CAudioManager::add_audio_data(mas_speech_data& data, long long time)
 349 {
 350         long long delta = mAudioRecordingDurationMilliseconds;
 351
 352         notify_audio_data_recording(time, data.buffer, data.len);
 353
 354         mas_speech_data_with_time data_with_time;
 355         data_with_time.data = data;
 356         data_with_time.time = time;
 357
 358         static unsigned int num = 0;
 359         const std::chrono::seconds interval(3);
 360         static auto last = std::chrono::steady_clock::now();
 361         auto now = std::chrono::steady_clock::now();
 362         if (now - last > interval) {
 363                 MWR_LOGE("Feeding audio data : num(%d), now(%" PRId64 "), %d %zu",
 364                         num, now.time_since_epoch().count(),
 365                         mStreamingThreadActive.load(), mAudioData.size());
 366                 last = now;
 367         }
 368         num++;
 369
 370         lock_guard<mutex> lock(mMutex);
 371
 372         /* Pop items only when the streaming is not activated */
 373         if (!mStreamingThreadActive.load()) {
 374                 while(false == mAudioData.empty() && mAudioData.front().time < time - delta) {
 375                         const auto &front = mAudioData.front();
 376                         if (front.data.buffer) {
 377                                 vm_free_simple(front.data.buffer);
 378                         }
 379                         mAudioData.pop_front();
 380                 }
 381         }
 382         validate_audio_data_event_field(data);
 383         mAudioData.push_back(data_with_time);
 384 }
 385
 386 void CAudioManager::feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len)
 387 {
 388         if (NULL == buffer || 0 == len) return;
 389
 390         mas_speech_data speech_data;
 391         speech_data.buffer = vm_malloc_simple(len);
 392         if (speech_data.buffer) {
 393                 long long time = get_current_milliseconds_after_epoch();
 394
 395                 speech_data.event = event;
 396                 speech_data.len = len;
 397                 memcpy(speech_data.buffer, buffer, len);
 398                 add_audio_data(speech_data, time);
 399         }
 400 }
 401
 402 void CAudioManager::finalize_audio_data()
 403 {
 404         MWR_LOGI("Adding FINISH event to audio data list");
 405         unsigned char final_buffer[2] = {'\0', };
 406         mas_speech_data speech_data;
 407         speech_data.event = MAS_SPEECH_STREAMING_EVENT_FINISH;
 408         speech_data.len = sizeof(final_buffer);
 409         speech_data.buffer = vm_malloc_simple(speech_data.len);
 410         if (speech_data.buffer) {
 411                 long long time = get_current_milliseconds_after_epoch();
 412
 413                 memcpy(speech_data.buffer, final_buffer, speech_data.len);
 414                 add_audio_data(speech_data, time);
 415         }
 416 }
 417
 418 void CAudioManager::clear_audio_data()
 419 {
 420         lock_guard<mutex> lock(mMutex);
 421         while(!mAudioData.empty()) {
 422                 const auto &front = mAudioData.front();
 423                 if (front.data.buffer) {
 424                         vm_free_simple(front.data.buffer);
 425                 }
 426                 mAudioData.pop_front();
 427         }
 428         mAudioData.clear();
 429 }
 430
 431 void CAudioManager::notify_audio_data_recording(long time, void* data, int len)
 432 {
 433         unique_lock<mutex> lock(mMutex, defer_lock);
 434         lock.lock();
 435         vector<IAudioEventObserver*> observers = mObservers;
 436         lock.unlock();
 437         for (const auto& observer : observers) {
 438                 if (observer) {
 439                         if (!observer->on_recording_audio_data(time, data, len)) {
 440                                 LOGE("[Recorder WARNING] One of the observer returned false");
 441                         }
 442                 }
 443         }
 444 }
 445
 446 void CAudioManager::start_streaming_current_utterance_data(long long start_time)
 447 {
 448         MWR_LOGI("[ENTER]");
 449         if (mStreamingThread.joinable()) {
 450                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 451                 return;
 452         }
 453         lock_guard<mutex> lock(mMutex);
 454         mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, start_time);
 455 }
 456
 457 void CAudioManager::stop_streaming_current_utterance_data()
 458 {
 459         MWR_LOGI("[ENTER]");
 460         if (mStreamingThread.joinable()) {
 461                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 462                 mStopStreamingThread.store(true);
 463                 try {
 464                         mStreamingThread.join();
 465                 } catch (std::exception &e) {
 466                         MWR_LOGE("Exception thrown : %s", e.what());
 467                 }
 468         }
 469         mStopStreamingThread.store(false);
 470
 471         /* FIXME : Need to move all the speech data to previous speech data for later use */
 472 }
 473
 474 void CAudioManager::start_streaming_previous_utterance_data()
 475 {
 476         MWR_LOGI("[ENTER]");
 477         if (mStreamingPreviousThread.joinable()) {
 478                 MWR_LOGE("ERROR : mStreamingPreviousThread is joinable, will not start a new thread");
 479                 return;
 480         }
 481         lock_guard<mutex> lock(mMutex);
 482         mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_audio_data_thread_func, this);
 483 }
 484
 485 void CAudioManager::stop_streaming_previous_utterance_data()
 486 {
 487         MWR_LOGI("[ENTER]");
 488         if (mStreamingPreviousThread.joinable()) {
 489                 MWR_LOGD("mStreamingPreviousThread is joinable, trying join()");
 490                 mStopStreamingPreviousThread.store(true);
 491                 try {
 492                         mStreamingPreviousThread.join();
 493                 } catch (std::exception &e) {
 494                         MWR_LOGE("Exception thrown : %s", e.what());
 495                 }
 496         }
 497         mStopStreamingThread.store(false);
 498 }
 499
 500 void CAudioManager::start_streaming_follow_up_data()
 501 {
 502         MWR_LOGI("[ENTER]");
 503         if (mStreamingThread.joinable()) {
 504                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 505                 return;
 506         }
 507
 508         mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, 0);
 509 }
 510
 511 void CAudioManager::stop_streaming_follow_up_data()
 512 {
 513         MWR_LOGI("[ENTER]");
 514         if (mStreamingThread.joinable()) {
 515                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 516                 mStopStreamingThread.store(true);
 517                 mStreamingThread.join();
 518         }
 519         mStopStreamingThread.store(false);
 520 }
 521
 522 void CAudioManager::set_background_volume(double ratio)
 523 {
 524         dependency_resolver_set_background_volume(ratio);
 525 }
 526
 527 } // wakeup
 528 } // multiassistant