plugins/wakeup-manager/src/wakeup_audio_manager.cpp

   1 #include "wakeup_audio_manager.h"
   2 #include "wakeup_manager_main.h"
   3 #include "dependency_resolver.h"
   4 #include "heap_tracer.h"
   5
   6 #include <algorithm>
   7
   8 #include <Ecore.h>
   9 #include <audio_io.h>
  10 #include <sound_manager.h>
  11 #include <sound_manager_internal.h>
  12
  13 namespace multiassistant
  14 {
  15 namespace wakeup
  16 {
  17
  18 /* Need to check whether this value needs to be configurable */
  19 static int g_speech_pcm_wait_count = 800;
  20
  21 static long long get_current_milliseconds_after_epoch()
  22 {
  23         auto now = chrono::steady_clock::now();
  24         auto now_ms = chrono::time_point_cast<chrono::milliseconds>(now);
  25         /* number of milliseconds since the epoch of system_clock */
  26         auto value = now_ms.time_since_epoch();
  27
  28         return value.count();
  29 }
  30
  31 CAudioManager::CAudioManager()
  32 {
  33 }
  34
  35 CAudioManager::~CAudioManager()
  36 {
  37 }
  38
  39 CAudioManager::CAudioManager(IAudioEventObserver *observer) : CAudioManager()
  40 {
  41         subscribe(observer);
  42 }
  43
  44 void recording_focus_state_watch_cb(int id, sound_stream_focus_mask_e focus_mask,
  45         sound_stream_focus_state_e focus_state, sound_stream_focus_change_reason_e reason,
  46         const char *extra_info, void *user_data)
  47 {
  48         MWR_LOGD("[Recorder] focus_mask : %d, focus_state : %d, reason : %d, [%s]",
  49                 focus_mask, focus_state, reason, extra_info);
  50
  51         if (nullptr == user_data) return;
  52
  53         /* The API description states that calling sound_manager_get_current_recording_focus()
  54                 function inside sound_stream_focus_state_watch_cb() function is prohibited */
  55         ecore_main_loop_thread_safe_call_async(
  56                 [](void* data) {
  57                         CAudioManager *manager = static_cast<CAudioManager*>(data);
  58                         if (manager) {
  59                                 manager->sound_focus_changed();
  60                         }
  61                 }, user_data);
  62 }
  63
  64 int CAudioManager::initialize(void)
  65 {
  66         sound_manager_add_focus_state_watch_cb(SOUND_STREAM_FOCUS_FOR_RECORDING,
  67                 recording_focus_state_watch_cb, this, &mSoundFocusWatchId);
  68         return 0;
  69 }
  70
  71 int CAudioManager::deinitialize(void)
  72 {
  73         clear_audio_data();
  74         if (mStreamingThread.joinable()) {
  75                 MWR_LOGD("mStreamingThread is joinable, trying join()");
  76                 mStopStreamingThread.store(true);
  77                 try {
  78                         mStreamingThread.join();
  79                 } catch (std::exception &e) {
  80                         MWR_LOGE("Exception thrown : %s", e.what());
  81                 }
  82         }
  83         mStopStreamingThread.store(false);
  84
  85         sound_manager_remove_focus_state_watch_cb(mSoundFocusWatchId);
  86
  87         return 0;
  88 }
  89
  90 void CAudioManager::sound_focus_changed()
  91 {
  92         sound_stream_focus_change_reason_e acquired_by;
  93         int sound_behavior;
  94         char* extra_info = NULL;
  95         int focus = sound_manager_get_current_recording_focus(&acquired_by, &sound_behavior, &extra_info);
  96         MWR_LOGD("[Recorder] sound focus has changed : %d %d %d %s", focus,
  97                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? acquired_by : -1),
  98                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? sound_behavior : -1),
  99                 (SOUND_MANAGER_ERROR_NO_DATA != focus ? extra_info : ""));
 100         if (SOUND_MANAGER_ERROR_NO_DATA == focus) {
 101                 if (mRecordingRequired && !mIsRecording) {
 102                         MWR_LOGD("[Recorder] Currently no other process has acquired sound focus, start recording");
 103                         start_recording(false);
 104                 }
 105         } else {
 106                 if (mIsRecording) {
 107                         MWR_LOGW("[Recorder] Sound focus acquired by other process, stop recording");
 108                         stop_recording(false);
 109                 }
 110         }
 111         if (extra_info) {
 112                 free(extra_info);
 113                 extra_info = NULL;
 114         }
 115 }
 116
 117 void CAudioManager::subscribe(IAudioEventObserver *observer)
 118 {
 119         if (observer) {
 120                 mObservers.push_back(observer);
 121         }
 122 }
 123
 124 void CAudioManager::unsubscribe(IAudioEventObserver *observer)
 125 {
 126         auto iter = find(mObservers.begin(), mObservers.end(), observer);
 127         if (iter != mObservers.end()) {
 128                 mObservers.erase(iter);
 129         }
 130 }
 131
 132 void CAudioManager::stop_recording(bool proactive)
 133 {
 134         dependency_resolver_stop_recording();
 135         if (proactive) {
 136                 mRecordingRequired = false;
 137         }
 138         mIsRecording = false;
 139 }
 140
 141 void CAudioManager::start_recording(bool proactive)
 142 {
 143         if (mIsRecording) {
 144                 stop_recording(false);
 145         }
 146         if (proactive) {
 147                 mRecordingRequired = true;
 148         }
 149
 150         sound_stream_focus_change_reason_e acquired_by;
 151         int sound_behavior;
 152         char* extra_info = NULL;
 153         if (SOUND_MANAGER_ERROR_NO_DATA == sound_manager_get_current_recording_focus(&acquired_by, &sound_behavior, &extra_info)) {
 154                 MWR_LOGD("[Recorder] Currently no other process has acquired sound focus, start recording");
 155                 dependency_resolver_start_recording();
 156                 mIsRecording = true;
 157         } else {
 158                 MWR_LOGW("[Recorder] Currently sound focus is acquired by other process, skip recording");
 159         }
 160         if (extra_info) {
 161                 free(extra_info);
 162                 extra_info = NULL;
 163         }
 164 }
 165
 166 void CAudioManager::set_recording_session(recording_session session)
 167 {
 168         dependency_resolver_set_recording_session((unsigned int)session);
 169 }
 170
 171 /* Need to consider adapting conventional producer-consumer model */
 172 void CAudioManager::streaming_previous_audio_data_thread_func()
 173 {
 174         MWR_LOGI("[ENTER]");
 175
 176         unique_lock<mutex> lock(mMutex, defer_lock);
 177
 178         /* get feedback data */
 179         size_t audio_data_size = 0;
 180         lock.lock();
 181         audio_data_size = mPreviousAudioData.size();
 182         for (int index = 0; index < audio_data_size; index++) {
 183                 mas_speech_data& speech_data = mPreviousAudioData.at(index).data;
 184                 for (const auto& observer : mObservers) {
 185                         if (observer) {
 186                                 if (!observer->on_streaming_audio_data(
 187                                         speech_data.event, speech_data.buffer, speech_data.len)) {
 188                                         LOGE("[Recorder WARNING] One of the observer returned false");
 189                                 }
 190                         }
 191
 192                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 193                                 MWR_LOGI("[INFO] Finish to send previous speech data");
 194                                 return;
 195                         }
 196                 }
 197
 198                 if (mStopStreamingThread.load()) {
 199                         MWR_LOGI("[INFO] Stop Streaming Requested, returning");
 200                         return;
 201                 }
 202         }
 203         lock.unlock();
 204
 205         MWR_LOGI("[EXIT]");
 206 }
 207
 208 void CAudioManager::streaming_audio_data_thread_func(long long start_time)
 209 {
 210         MWR_LOGI("[ENTER]");
 211
 212         unique_lock<mutex> lock(mMutex, defer_lock);
 213         bool finish_event_sent = false;
 214
 215         lock.lock();
 216         auto lead = mAudioData.begin();
 217         auto iter = lead;
 218         while (lead != mAudioData.end() && lead->time < start_time) {
 219                 iter = lead;
 220                 advance(lead, 1);
 221         }
 222         MWR_LOGI("data_count : %zu", mAudioData.size());
 223         lock.unlock();
 224
 225         while (!(mStopStreamingThread.load())) {
 226                 int ret = -1;
 227                 int cnt = 0;
 228
 229                 /* get feedback data */
 230                 lock.lock();
 231                 auto end = mAudioData.end();
 232                 lock.unlock();
 233                 if (lead == end) {
 234                         /* empty queue */
 235                         MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
 236
 237                         /* waiting */
 238                         while (!(mStopStreamingThread.load())) {
 239                                 this_thread::sleep_for(chrono::milliseconds(10));
 240                                 lock.lock();
 241                                 end = mAudioData.end();
 242                                 auto begin = mAudioData.begin();
 243                                 lock.unlock();
 244                                 if (iter == end) {
 245                                         lead = begin;
 246                                 } else {
 247                                         lead = iter;
 248                                         if (lead != end) {
 249                                                 advance(lead, 1);
 250                                         }
 251                                 }
 252                                 if (lead != end) {
 253                                         MWR_LOGD("[INFO] Resume thread");
 254                                         break;
 255                                 }
 256                                 if (g_speech_pcm_wait_count < cnt) {
 257                                         unsigned char final_buffer[2] = {'\0', };
 258                                         MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
 259                                         for (const auto& observer : mObservers) {
 260                                                 if (observer) {
 261                                                         if (!observer->on_streaming_audio_data(
 262                                                                 MAS_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
 263                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 264                                                         }
 265                                                         if (!observer->on_streaming_audio_data(
 266                                                                 MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 267                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 268                                                         }
 269                                                 }
 270                                         }
 271                                         return;
 272                                 }
 273                                 cnt++;
 274                         }
 275                         MWR_LOGD("[INFO] Finish to wait for new feedback data come");
 276
 277                         /* resume feedback thread */
 278                         continue;
 279                 }
 280
 281                 /* FIXME : Extracted audio data here should be used as previous audio data*/
 282
 283                 lock.lock();
 284                 end = mAudioData.end();
 285
 286                 if (lead != end) {
 287                         iter = lead;
 288                         mas_speech_data& speech_data = iter->data;
 289                         for (const auto& observer : mObservers) {
 290                                 if (observer) {
 291                                         if (!observer->on_streaming_audio_data(
 292                                                 speech_data.event, speech_data.buffer, speech_data.len)) {
 293                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 294                                         }
 295                                 }
 296                         }
 297
 298                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 299                                 MWR_LOGI("[INFO] Finish to get and send speech data");
 300                                 finish_event_sent = true;
 301                                 break;
 302                         }
 303
 304                         advance(lead, 1);
 305                 }
 306                 lock.unlock();
 307         }
 308
 309         if (true != finish_event_sent) {
 310                 unsigned char final_buffer[2] = {'\0', };
 311                 for (const auto& observer : mObservers) {
 312                         if (observer) {
 313                                 if (!observer->on_streaming_audio_data(
 314                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 315                                         LOGE("[Recorder WARNING] One of the observer returned false");
 316                                 }
 317                         }
 318                 }
 319         }
 320         MWR_LOGI("[EXIT]");
 321 }
 322
 323 void CAudioManager::add_audio_data(mas_speech_data& data, long long time)
 324 {
 325         long long delta = mAudioRecordingDurationMilliseconds;
 326
 327         notify_audio_data_recording(time, data.buffer, data.len);
 328
 329         mas_speech_data_with_time data_with_time;
 330         data_with_time.data = data;
 331         data_with_time.time = time;
 332
 333         lock_guard<mutex> lock(mMutex);
 334
 335         /* Pop items only when the streaming is not activated */
 336         while(false == mAudioData.empty() && mAudioData.front().time < time - delta) {
 337                 const auto &front = mAudioData.front();
 338                 if (front.data.buffer) {
 339                         vm_free_simple(front.data.buffer);
 340                 }
 341                 mAudioData.pop_front();
 342         }
 343         mAudioData.push_back(data_with_time);
 344 }
 345
 346 void CAudioManager::feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len)
 347 {
 348         if (NULL == buffer || 0 == len) return;
 349
 350         mas_speech_data speech_data;
 351         speech_data.buffer = vm_malloc_simple(len);
 352         if (speech_data.buffer) {
 353                 long long time = get_current_milliseconds_after_epoch();
 354
 355                 speech_data.event = event;
 356                 speech_data.len = len;
 357                 memcpy(speech_data.buffer, buffer, len);
 358                 add_audio_data(speech_data, time);
 359         }
 360 }
 361
 362 void CAudioManager::finalize_audio_data()
 363 {
 364         unsigned char final_buffer[2] = {'\0', };
 365         mas_speech_data speech_data;
 366         speech_data.event = MAS_SPEECH_STREAMING_EVENT_FINISH;
 367         speech_data.len = sizeof(final_buffer);
 368         speech_data.buffer = vm_malloc_simple(speech_data.len);
 369         if (speech_data.buffer) {
 370                 long long time = get_current_milliseconds_after_epoch();
 371
 372                 memcpy(speech_data.buffer, final_buffer, speech_data.len);
 373                 add_audio_data(speech_data, time);
 374         }
 375 }
 376
 377 void CAudioManager::clear_audio_data()
 378 {
 379         lock_guard<mutex> lock(mMutex);
 380         while(!mAudioData.empty()) {
 381                 const auto &front = mAudioData.front();
 382                 if (front.data.buffer) {
 383                         vm_free_simple(front.data.buffer);
 384                 }
 385                 mAudioData.pop_front();
 386         }
 387         mAudioData.clear();
 388 }
 389
 390 void CAudioManager::notify_audio_data_recording(long time, void* data, int len)
 391 {
 392         for (const auto& observer : mObservers) {
 393                 if (observer) {
 394                         if (!observer->on_recording_audio_data(time, data, len)) {
 395                                 LOGE("[Recorder WARNING] One of the observer returned false");
 396                         }
 397                 }
 398         }
 399 }
 400
 401 void CAudioManager::start_streaming_current_utterance_data(long long start_time)
 402 {
 403         if (mStreamingThread.joinable()) {
 404                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 405                 return;
 406         }
 407         lock_guard<mutex> lock(mMutex);
 408         mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, start_time);
 409 }
 410
 411 void CAudioManager::stop_streaming_current_utterance_data()
 412 {
 413         if (mStreamingThread.joinable()) {
 414                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 415                 mStopStreamingThread.store(true);
 416                 try {
 417                         mStreamingThread.join();
 418                 } catch (std::exception &e) {
 419                         MWR_LOGE("Exception thrown : %s", e.what());
 420                 }
 421         }
 422         mStopStreamingThread.store(false);
 423
 424         /* FIXME : Need to move all the speech data to previous speech data for later use */
 425 }
 426
 427 void CAudioManager::start_streaming_previous_utterance_data()
 428 {
 429         if (mStreamingPreviousThread.joinable()) {
 430                 MWR_LOGE("ERROR : mStreamingPreviousThread is joinable, will not start a new thread");
 431                 return;
 432         }
 433         lock_guard<mutex> lock(mMutex);
 434         mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_audio_data_thread_func, this);
 435 }
 436
 437 void CAudioManager::stop_streaming_previous_utterance_data()
 438 {
 439         if (mStreamingPreviousThread.joinable()) {
 440                 MWR_LOGD("mStreamingPreviousThread is joinable, trying join()");
 441                 mStopStreamingPreviousThread.store(true);
 442                 try {
 443                         mStreamingPreviousThread.join();
 444                 } catch (std::exception &e) {
 445                         MWR_LOGE("Exception thrown : %s", e.what());
 446                 }
 447         }
 448         mStopStreamingThread.store(false);
 449 }
 450
 451 void CAudioManager::start_streaming_follow_up_data()
 452 {
 453         if (mStreamingThread.joinable()) {
 454                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 455                 return;
 456         }
 457
 458         mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, 0);
 459 }
 460
 461 void CAudioManager::stop_streaming_follow_up_data()
 462 {
 463         if (mStreamingThread.joinable()) {
 464                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 465                 mStopStreamingThread.store(true);
 466                 mStreamingThread.join();
 467         }
 468         mStopStreamingThread.store(false);
 469 }
 470
 471 void CAudioManager::set_background_volume(double ratio)
 472 {
 473         dependency_resolver_set_background_volume(ratio);
 474 }
 475
 476 } // wakeup
 477 } // multiassistant