2 * Copyright (c) 2011-2016 Samsung Electronics Co., Ltd All Rights Reserved
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 * http://www.apache.org/licenses/LICENSE-2.0
7 * Unless required by applicable law or agreed to in writing, software
8 * distributed under the License is distributed on an "AS IS" BASIS,
9 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 * See the License for the specific language governing permissions and
11 * limitations under the License.
22 #include <sound_manager.h>
24 #include <bluetooth.h>
28 #include "sttd_dbus.h"
29 #include "sttd_recorder.h"
30 #include "sttd_main.h"
34 #define FRAME_LENGTH 160
35 #define BUFFER_LENGTH FRAME_LENGTH * 2
37 static pthread_mutex_t sttd_audio_in_handle_mutex = PTHREAD_MUTEX_INITIALIZER;
40 STTD_RECORDER_STATE_NONE = -1,
41 STTD_RECORDER_STATE_READY = 0, /**< Recorder is ready to start */
42 STTD_RECORDER_STATE_RECORDING /**< In the middle of recording */
43 } sttd_recorder_state;
49 sttp_audio_type_e audio_type;
52 static GSList *g_recorder_list;
54 static int g_recording_engine_id;
56 static stt_recorder_audio_cb g_audio_cb;
58 static sound_stream_info_h g_stream_info_h;
60 static stt_recorder_interrupt_cb g_interrupt_cb;
62 static sttd_recorder_state g_recorder_state = STTD_RECORDER_STATE_NONE;
64 static int g_buffer_count;
66 /* Sound buf save for test */
71 static char g_temp_file_name[128] = {'\0',};
75 static int g_count = 1;
79 static float get_volume_decibel(char* data, int size, sttp_audio_type_e type);
81 static stt_recorder_s* __get_recorder(int engine_id);
83 static int g_bt_extend_count;
85 #define SMART_CONTROL_EXTEND_CMD 0x03
86 #define SMART_CONTROL_START_CMD 0x04
88 static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data)
90 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address);
93 static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data)
95 if (STTD_RECORDER_STATE_RECORDING != g_recorder_state) {
96 /* SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Exit audio reading normal func"); */
100 if (NULL != g_audio_cb) {
101 if (0 != g_audio_cb((void*)voice_data->audio_buf, (unsigned int)voice_data->length)) {
102 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to read audio");
103 sttd_recorder_stop(g_recording_engine_id);
106 stt_recorder_s* recorder;
107 recorder = __get_recorder(g_recording_engine_id);
108 if (NULL == recorder) {
112 float vol_db = get_volume_decibel((char*)voice_data->audio_buf, (int)voice_data->length, recorder->audio_type);
113 if (0 != sttdc_send_set_volume(recorder->uid, vol_db)) {
114 SLOG(LOG_ERROR, TAG_STTD, "[Recorder] Fail to send recording volume(%f)", vol_db);
118 if (0 == g_buffer_count || 0 == g_buffer_count % 50) {
119 SLOG(LOG_WARN, TAG_STTD, "[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length);
121 if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) {
122 const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 };
123 if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) {
124 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_send_rc_command(NULL, %s, %d)", input_data, sizeof(input_data));
126 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Extend bt audio recorder");
131 if (100000 == g_buffer_count) {
139 /* write pcm buffer */
140 fwrite(data, 1, len, g_pFile);
147 const char* __stt_get_focus_changed_reason_code(sound_stream_focus_change_reason_e reason)
150 case SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA: return "SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA";
151 case SOUND_STREAM_FOCUS_CHANGED_BY_SYSTEM: return "SOUND_STREAM_FOCUS_CHANGED_BY_SYSTEM";
152 case SOUND_STREAM_FOCUS_CHANGED_BY_ALARM: return "SOUND_STREAM_FOCUS_CHANGED_BY_ALARM";
153 case SOUND_STREAM_FOCUS_CHANGED_BY_NOTIFICATION: return "SOUND_STREAM_FOCUS_CHANGED_BY_NOTIFICATION";
154 case SOUND_STREAM_FOCUS_CHANGED_BY_EMERGENCY: return "SOUND_STREAM_FOCUS_CHANGED_BY_EMERGENCY";
155 case SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_INFORMATION: return "SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_INFORMATION";
156 case SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_RECOGNITION: return "SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_RECOGNITION";
157 case SOUND_STREAM_FOCUS_CHANGED_BY_RINGTONE: return "SOUND_STREAM_FOCUS_CHANGED_BY_RINGTONE";
158 case SOUND_STREAM_FOCUS_CHANGED_BY_VOIP: return "SOUND_STREAM_FOCUS_CHANGED_BY_VOIP";
159 case SOUND_STREAM_FOCUS_CHANGED_BY_CALL: return "SOUND_STREAM_FOCUS_CHANGED_BY_CALL";
160 case SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA_EXTERNAL_ONLY: return "SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA_EXTERNAL_ONLY";
161 default: return "Undefined reason code";
165 void __recorder_focus_state_cb(sound_stream_info_h stream_info, sound_stream_focus_change_reason_e reason, const char *extra_info, void *user_data)
167 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Focus state changed cb");
169 if (stream_info != g_stream_info_h) {
170 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Invalid stream info handle");
175 sound_stream_focus_state_e state_for_recording;
176 ret = sound_manager_get_focus_state(g_stream_info_h, NULL, &state_for_recording);
177 if (SOUND_MANAGER_ERROR_NONE != ret) {
178 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to get focus state");
182 SLOG(LOG_WARN, TAG_STTD, "[Recorder] focus state changed to (%d) with reason(%s)", (int)state_for_recording, __stt_get_focus_changed_reason_code(reason));
184 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state && SOUND_STREAM_FOCUS_STATE_RELEASED == state_for_recording) {
185 SLOG(LOG_WARN, TAG_STTD, "[Recorder] Focus released as interrupt");
186 if (NULL != g_interrupt_cb) {
193 int sttd_recorder_initialize(stt_recorder_audio_cb audio_cb, stt_recorder_interrupt_cb interrupt_cb)
195 if (NULL == audio_cb || NULL == interrupt_cb) {
196 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Input param is NOT valid");
197 return STTD_ERROR_INVALID_PARAMETER;
200 if (STTD_RECORDER_STATE_NONE != g_recorder_state) {
201 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Current state of recorder is recording");
202 return STTD_ERROR_INVALID_STATE;
205 if (0 != pthread_mutex_init(&sttd_audio_in_handle_mutex, NULL)) {
206 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to initialize audio in handle mutex.");
209 g_audio_cb = audio_cb;
210 g_interrupt_cb = interrupt_cb;
211 g_recorder_state = STTD_RECORDER_STATE_NONE;
212 g_recording_engine_id = -1;
214 if (0 != sound_manager_create_stream_information(SOUND_STREAM_TYPE_VOICE_RECOGNITION, __recorder_focus_state_cb, NULL, &g_stream_info_h)) {
215 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to create stream info");
219 if (BT_ERROR_NONE != bt_initialize()) {
220 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to init bt");
221 return STTD_ERROR_OPERATION_FAILED;
224 if (BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, NULL)) {
225 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_host_initialize()");
226 return STTD_ERROR_OPERATION_FAILED;
233 int sttd_recorder_deinitialize()
235 if (0 != pthread_mutex_destroy(&sttd_audio_in_handle_mutex)) {
236 SLOG(LOG_ERROR, TAG_STTD, "[Server ERROR] Fail to destroy audio in handle mutex.");
239 if (0 != sound_manager_destroy_stream_information(g_stream_info_h)) {
240 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to destroy stream info");
243 /* Remove all recorder */
245 stt_recorder_s *recorder = NULL;
247 iter = g_slist_nth(g_recorder_list, 0);
249 while (NULL != iter) {
250 recorder = iter->data;
252 if (NULL != recorder) {
253 g_recorder_list = g_slist_remove(g_recorder_list, recorder);
254 if(recorder->audio_h) {
255 audio_in_destroy(recorder->audio_h);
256 recorder->audio_h = NULL;
261 iter = g_slist_nth(g_recorder_list, 0);
265 bt_hid_host_deinitialize ();
270 g_recorder_state = STTD_RECORDER_STATE_NONE;
275 static stt_recorder_s* __get_recorder(int engine_id)
278 stt_recorder_s *recorder = NULL;
280 iter = g_slist_nth(g_recorder_list, 0);
282 while (NULL != iter) {
283 recorder = iter->data;
285 if (recorder->engine_id == engine_id) {
289 iter = g_slist_next(iter);
295 int sttd_recorder_set_audio_session()
300 int sttd_recorder_unset_audio_session()
305 int sttd_recorder_create(int engine_id, int uid, sttp_audio_type_e type, int channel, unsigned int sample_rate)
307 /* Check engine id is valid */
308 if (NULL != __get_recorder(engine_id)) {
309 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is already registered");
310 return STTD_ERROR_INVALID_PARAMETER;
313 audio_in_h temp_in_h = NULL;
316 audio_channel_e audio_ch;
317 audio_sample_type_e audio_type;
320 case 1: audio_ch = AUDIO_CHANNEL_MONO; break;
321 case 2: audio_ch = AUDIO_CHANNEL_STEREO; break;
323 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Input channel is not supported");
324 return STTD_ERROR_OPERATION_FAILED;
329 case STTP_AUDIO_TYPE_PCM_S16_LE: audio_type = AUDIO_SAMPLE_TYPE_S16_LE; break;
330 case STTP_AUDIO_TYPE_PCM_U8: audio_type = AUDIO_SAMPLE_TYPE_U8; break;
332 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Invalid Audio Type");
333 return STTD_ERROR_OPERATION_FAILED;
338 ret = audio_in_create(sample_rate, audio_ch, audio_type, &temp_in_h);
339 if (AUDIO_IO_ERROR_NONE != ret) {
340 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to create audio handle : %d", ret);
341 return STTD_ERROR_OPERATION_FAILED;
344 if (BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, NULL)) {
345 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()");
346 return STTD_ERROR_OPERATION_FAILED;
350 stt_recorder_s* recorder;
351 recorder = (stt_recorder_s*)calloc(1, sizeof(stt_recorder_s));
352 if (NULL == recorder) {
355 audio_in_destroy(temp_in_h);
359 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to allocate memory");
360 return STTD_ERROR_OUT_OF_MEMORY;
363 recorder->engine_id = engine_id;
365 recorder->audio_h = temp_in_h;
366 recorder->audio_type = type;
368 g_recorder_list = g_slist_append(g_recorder_list, recorder);
370 g_recorder_state = STTD_RECORDER_STATE_READY;
375 int sttd_recorder_destroy(int engine_id)
377 // critical section required because this function can be called from stt engine thread context
378 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Enter critical section");
379 pthread_mutex_lock(&sttd_audio_in_handle_mutex);
381 /* Check engine id is valid */
382 stt_recorder_s* recorder;
383 recorder = __get_recorder(engine_id);
384 if (NULL == recorder) {
385 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
386 pthread_mutex_unlock(&sttd_audio_in_handle_mutex);
387 return STTD_ERROR_INVALID_PARAMETER;
392 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state) {
393 if (recorder->audio_h) {
394 ret = audio_in_unprepare(recorder->audio_h);
395 if (AUDIO_IO_ERROR_NONE != ret) {
396 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to unprepare audioin : %d", ret);
400 g_recorder_state = STTD_RECORDER_STATE_READY;
403 if (recorder->audio_h) {
404 ret = audio_in_destroy(recorder->audio_h);
405 if (AUDIO_IO_ERROR_NONE != ret) {
406 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to destroy audioin : %d", ret);
408 recorder->audio_h = NULL;
411 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state) {
412 g_recorder_state = STTD_RECORDER_STATE_READY;
415 bt_hid_unset_audio_data_receive_cb();
418 g_recorder_list = g_slist_remove(g_recorder_list, recorder);
422 pthread_mutex_unlock(&sttd_audio_in_handle_mutex);
423 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Leave critical section");
428 static float get_volume_decibel(char* data, int size, sttp_audio_type_e type)
430 #define MAX_AMPLITUDE_MEAN_16 32768
431 #define MAX_AMPLITUDE_MEAN_08 128
438 unsigned long long square_sum = 0;
440 if (type == STTP_AUDIO_TYPE_PCM_S16_LE)
445 for (i = 0; i < size; i += (depthByte<<1)) {
446 if (depthByte == 2) {
448 memcpy(&pcm16, data + i, sizeof(short));
449 square_sum += pcm16 * pcm16;
452 memcpy(&pcm8, data + i, sizeof(char));
453 square_sum += pcm8 * pcm8;
461 rms = sqrt(square_sum/count);
464 db = 20 * log10(rms/MAX_AMPLITUDE_MEAN_16);
466 db = 20 * log10(rms/MAX_AMPLITUDE_MEAN_08);
472 Eina_Bool __read_audio_func(void *data)
475 static char g_buffer[BUFFER_LENGTH];
477 /* Check engine id is valid */
478 stt_recorder_s* recorder;
479 recorder = __get_recorder(g_recording_engine_id);
480 if (NULL == recorder) {
484 if (STTD_RECORDER_STATE_READY == g_recorder_state) {
485 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Exit audio reading func");
489 read_byte = audio_in_read(recorder->audio_h, g_buffer, BUFFER_LENGTH);
491 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Fail to read audio : %d", read_byte);
492 g_recorder_state = STTD_RECORDER_STATE_READY;
496 if (0 != g_audio_cb(g_buffer, read_byte)) {
497 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Fail audio callback");
498 sttd_recorder_stop(g_recording_engine_id);
502 if (0 == g_buffer_count % 30) {
503 float vol_db = get_volume_decibel(g_buffer, BUFFER_LENGTH, recorder->audio_type);
504 if (0 != sttdc_send_set_volume(recorder->uid, vol_db)) {
505 SLOG(LOG_ERROR, TAG_STTD, "[Recorder] Fail to send recording volume(%f)", vol_db);
510 if (0 == g_buffer_count % 50) {
511 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, read_byte);
513 if (100000 == g_buffer_count) {
521 /* write pcm buffer */
522 fwrite(g_buffer, 1, BUFFER_LENGTH, g_pFile);
529 int sttd_recorder_start(int engine_id)
531 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state)
536 /* Check engine id is valid */
537 stt_recorder_s* recorder;
538 recorder = __get_recorder(engine_id);
539 if (NULL == recorder) {
540 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
541 return STTD_ERROR_INVALID_PARAMETER;
544 ret = sound_manager_acquire_focus(g_stream_info_h, SOUND_STREAM_FOCUS_FOR_RECORDING, NULL);
545 if (SOUND_MANAGER_ERROR_NONE != ret) {
546 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to acquire focus : %d", ret);
548 ret = audio_in_set_stream_info(recorder->audio_h, g_stream_info_h);
549 if (AUDIO_IO_ERROR_NONE != ret) {
550 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to set stream info");
554 ret = audio_in_prepare(recorder->audio_h);
555 if (AUDIO_IO_ERROR_NONE != ret) {
556 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to start audio : %d", ret);
557 return STTD_ERROR_RECORDER_BUSY;
560 /* Add ecore timer to read audio data */
561 ecore_timer_add(0, __read_audio_func, NULL);
564 g_bt_extend_count = 0;
565 const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00 };
567 bool started = false;
568 while (5 > bt_retry) {
569 ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data));
570 if (BT_ERROR_NONE == ret) {
571 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Start bt audio recorder");
574 } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
575 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_send_rc_command(NULL, %s, %d)", input_data, sizeof(input_data));
582 if (false == started) {
583 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to start bt audio");
584 return STTD_ERROR_OPERATION_FAILED;
587 g_recorder_state = STTD_RECORDER_STATE_RECORDING;
588 g_recording_engine_id = engine_id;
595 snprintf(g_temp_file_name, sizeof(g_temp_file_name), "/tmp/stt_temp_%d_%d", getpid(), g_count);
596 SECURE_SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Temp file name=[%s]", g_temp_file_name);
599 g_pFile = fopen(g_temp_file_name, "wb+");
601 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] File not found!");
609 int sttd_recorder_stop(int engine_id)
611 if (STTD_RECORDER_STATE_READY == g_recorder_state)
614 /* Check engine id is valid */
615 stt_recorder_s* recorder;
616 recorder = __get_recorder(engine_id);
617 if (NULL == recorder) {
618 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
619 return STTD_ERROR_INVALID_PARAMETER;
624 ret = audio_in_unprepare(recorder->audio_h);
625 if (AUDIO_IO_ERROR_NONE != ret) {
626 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to unprepare audioin : %d", ret);
630 bool stopped = false;
631 while (5 > bt_retry) {
632 ret = bt_hid_rc_stop_sending_voice(NULL);
633 if (BT_ERROR_NONE == ret) {
634 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Stop bt audio");
637 } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
638 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_rc_stop_sending_voice()");
645 if (false == stopped) {
646 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to stop bt audio");
647 return STTD_ERROR_OPERATION_FAILED;
651 g_recorder_state = STTD_RECORDER_STATE_READY;
652 g_recording_engine_id = -1;
654 ret = sound_manager_release_focus(g_stream_info_h, SOUND_STREAM_FOCUS_FOR_RECORDING, NULL);
655 if (SOUND_MANAGER_ERROR_NONE != ret) {
656 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to release focus :%d", ret);