2 * Copyright (c) 2011-2016 Samsung Electronics Co., Ltd All Rights Reserved
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 * http://www.apache.org/licenses/LICENSE-2.0
7 * Unless required by applicable law or agreed to in writing, software
8 * distributed under the License is distributed on an "AS IS" BASIS,
9 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 * See the License for the specific language governing permissions and
11 * limitations under the License.
22 #include <sound_manager.h>
23 #include <sound_manager_internal.h>
25 #include <bluetooth_product.h>
29 #include "sttd_dbus.h"
30 #include "sttd_recorder.h"
31 #include "sttd_main.h"
34 #define FRAME_LENGTH 320
35 #define BUFFER_LENGTH FRAME_LENGTH * 2
37 static pthread_mutex_t sttd_audio_in_handle_mutex = PTHREAD_MUTEX_INITIALIZER;
40 static Ecore_Timer* g_audio_timer = NULL;
44 STTD_RECORDER_STATE_NONE = -1,
45 STTD_RECORDER_STATE_READY = 0, /**< Recorder is ready to start */
46 STTD_RECORDER_STATE_RECORDING /**< In the middle of recording */
47 } sttd_recorder_state;
52 stte_audio_type_e audio_type;
55 static stt_recorder_s* g_recorder = NULL;
57 static stt_recorder_audio_cb g_audio_cb;
59 static sound_stream_info_h g_stream_info_h;
61 static stt_recorder_interrupt_cb g_interrupt_cb;
63 static sttd_recorder_state g_recorder_state = STTD_RECORDER_STATE_NONE;
65 static int g_buffer_count;
67 static int g_stream_focus_id;
69 /* Sound buf save for test */
74 static char g_temp_file_name[128] = {'\0',};
78 static int g_count = 1;
82 static float get_volume_decibel(char* data, int size, stte_audio_type_e type);
84 //static stt_recorder_s* __get_recorder(int engine_id);
86 #define SMART_CONTROL_EXTEND_CMD 0x03
87 #define SMART_CONTROL_START_CMD 0x04
89 static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data)
91 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address);
94 static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data)
96 if (STTD_RECORDER_STATE_RECORDING != g_recorder_state) {
97 /* SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Exit audio reading normal func"); */
101 if (NULL != g_audio_cb) {
102 if (0 != g_audio_cb((void*)voice_data->audio_buf, (unsigned int)voice_data->length)) {
103 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to read audio");
104 sttd_recorder_stop();
107 if (NULL == g_recorder) {
111 float vol_db = get_volume_decibel((char*)voice_data->audio_buf, (int)voice_data->length, g_recorder->audio_type);
112 if (0 != sttdc_send_set_volume(g_recorder->uid, vol_db)) {
113 SLOG(LOG_ERROR, TAG_STTD, "[Recorder] Fail to send recording volume(%f)", vol_db);
117 if (0 == g_buffer_count || 0 == g_buffer_count % 50) {
118 SLOG(LOG_WARN, TAG_STTD, "[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length);
120 if (100000 == g_buffer_count) {
128 /* write pcm buffer */
129 fwrite(data, 1, len, g_pFile);
136 const char* __stt_get_focus_changed_reason_code(sound_stream_focus_change_reason_e reason)
139 case SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA: return "SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA";
140 case SOUND_STREAM_FOCUS_CHANGED_BY_SYSTEM: return "SOUND_STREAM_FOCUS_CHANGED_BY_SYSTEM";
141 case SOUND_STREAM_FOCUS_CHANGED_BY_ALARM: return "SOUND_STREAM_FOCUS_CHANGED_BY_ALARM";
142 case SOUND_STREAM_FOCUS_CHANGED_BY_NOTIFICATION: return "SOUND_STREAM_FOCUS_CHANGED_BY_NOTIFICATION";
143 case SOUND_STREAM_FOCUS_CHANGED_BY_EMERGENCY: return "SOUND_STREAM_FOCUS_CHANGED_BY_EMERGENCY";
144 case SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_INFORMATION: return "SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_INFORMATION";
145 case SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_RECOGNITION: return "SOUND_STREAM_FOCUS_CHANGED_BY_VOICE_RECOGNITION";
146 case SOUND_STREAM_FOCUS_CHANGED_BY_RINGTONE: return "SOUND_STREAM_FOCUS_CHANGED_BY_RINGTONE";
147 case SOUND_STREAM_FOCUS_CHANGED_BY_VOIP: return "SOUND_STREAM_FOCUS_CHANGED_BY_VOIP";
148 case SOUND_STREAM_FOCUS_CHANGED_BY_CALL: return "SOUND_STREAM_FOCUS_CHANGED_BY_CALL";
149 case SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA_EXTERNAL_ONLY: return "SOUND_STREAM_FOCUS_CHANGED_BY_MEDIA_EXTERNAL_ONLY";
150 default: return "Undefined reason code";
154 void __recorder_focus_state_watch_cb(int id, sound_stream_focus_mask_e focus_mask, sound_stream_focus_state_e focus_state,
155 sound_stream_focus_change_reason_e reason, const char *extra_info, void *user_data)
157 SLOG(LOG_WARN, TAG_STTD, "[Recorder] focus_state_watch_cb is called, focus_state(%d), reason(%s)", (int)focus_state, __stt_get_focus_changed_reason_code(reason));
159 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state && SOUND_STREAM_FOCUS_STATE_ACQUIRED == focus_state) {
160 SLOG(LOG_WARN, TAG_STTD, "[Recorder] Focus released as interrupt");
161 if (NULL != g_interrupt_cb) {
167 int sttd_recorder_initialize(stt_recorder_audio_cb audio_cb, stt_recorder_interrupt_cb interrupt_cb)
169 if (NULL == audio_cb || NULL == interrupt_cb) {
170 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Input param is NOT valid");
171 return STTD_ERROR_INVALID_PARAMETER;
174 if (STTD_RECORDER_STATE_NONE != g_recorder_state) {
175 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Current state of recorder is recording");
176 return STTD_ERROR_INVALID_STATE;
179 if (0 != pthread_mutex_init(&sttd_audio_in_handle_mutex, NULL)) {
180 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to initialize audio in handle mutex.");
183 g_audio_cb = audio_cb;
184 g_interrupt_cb = interrupt_cb;
185 g_recorder_state = STTD_RECORDER_STATE_NONE;
188 if (0 != sound_manager_create_stream_information_internal(SOUND_STREAM_TYPE_VOICE_RECOGNITION_SERVICE, NULL, NULL, &g_stream_info_h)) {
189 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to create stream info");
192 if (0 != sound_manager_add_focus_state_watch_cb(SOUND_STREAM_FOCUS_FOR_BOTH, __recorder_focus_state_watch_cb, NULL, &g_stream_focus_id)) {
193 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to add focus state watch callback");
197 if (BT_ERROR_NONE != bt_product_init()) {
198 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to init bt");
199 return STTD_ERROR_OPERATION_FAILED;
202 if (BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, NULL)) {
203 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_host_initialize()");
204 return STTD_ERROR_OPERATION_FAILED;
211 int sttd_recorder_deinitialize()
213 if (0 != pthread_mutex_destroy(&sttd_audio_in_handle_mutex)) {
214 SLOG(LOG_ERROR, TAG_STTD, "[Server ERROR] Fail to destroy audio in handle mutex.");
217 if (0 != sound_manager_remove_focus_state_watch_cb(g_stream_focus_id)) {
218 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to remove focus state watch callback");
221 if (0 != sound_manager_destroy_stream_information(g_stream_info_h)) {
222 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to destroy stream info");
225 if (NULL != g_recorder) {
226 audio_in_destroy(g_recorder->audio_h);
227 g_recorder->audio_h = NULL;
232 #ifdef __UNUSED_CODES__
233 /* Remove all recorder */
235 stt_recorder_s *recorder = NULL;
237 iter = g_slist_nth(g_recorder_list, 0);
239 while (NULL != iter) {
240 recorder = iter->data;
242 if (NULL != recorder) {
243 g_recorder_list = g_slist_remove(g_recorder_list, recorder);
244 if (recorder->audio_h) {
245 audio_in_destroy(recorder->audio_h);
246 recorder->audio_h = NULL;
251 iter = g_slist_nth(g_recorder_list, 0);
256 bt_hid_host_deinitialize();
261 g_recorder_state = STTD_RECORDER_STATE_NONE;
266 int sttd_recorder_set_audio_session()
271 int sttd_recorder_unset_audio_session()
276 int sttd_recorder_create(stte_audio_type_e type, int channel, unsigned int sample_rate)
278 /* Check engine id is valid */
279 if (NULL != g_recorder) {
280 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is already registered");
281 return STTD_ERROR_INVALID_PARAMETER;
284 audio_in_h temp_in_h = NULL;
287 audio_channel_e audio_ch;
288 audio_sample_type_e audio_type;
291 case 1: audio_ch = AUDIO_CHANNEL_MONO; break;
292 case 2: audio_ch = AUDIO_CHANNEL_STEREO; break;
294 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Input channel is not supported");
295 return STTD_ERROR_OPERATION_FAILED;
300 case STTE_AUDIO_TYPE_PCM_S16_LE: audio_type = AUDIO_SAMPLE_TYPE_S16_LE; break;
301 case STTE_AUDIO_TYPE_PCM_U8: audio_type = AUDIO_SAMPLE_TYPE_U8; break;
303 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Invalid Audio Type");
304 return STTD_ERROR_OPERATION_FAILED;
309 ret = audio_in_create(sample_rate, audio_ch, audio_type, &temp_in_h);
310 if (AUDIO_IO_ERROR_NONE != ret) {
311 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to create audio handle : %d", ret);
312 return STTD_ERROR_OPERATION_FAILED;
315 if (BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, NULL)) {
316 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()");
317 return STTD_ERROR_OPERATION_FAILED;
321 stt_recorder_s* recorder;
322 recorder = (stt_recorder_s*)calloc(1, sizeof(stt_recorder_s));
323 if (NULL == recorder) {
326 audio_in_destroy(temp_in_h);
330 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to allocate memory");
331 return STTD_ERROR_OUT_OF_MEMORY;
335 recorder->audio_h = temp_in_h;
336 recorder->audio_type = type;
338 g_recorder = recorder;
340 g_recorder_state = STTD_RECORDER_STATE_READY;
345 int sttd_recorder_destroy()
347 // critical section required because this function can be called from stt engine thread context
348 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Enter critical section");
349 pthread_mutex_lock(&sttd_audio_in_handle_mutex);
351 /* Check engine id is valid */
352 if (NULL == g_recorder) {
353 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
354 pthread_mutex_unlock(&sttd_audio_in_handle_mutex);
355 return STTD_ERROR_INVALID_PARAMETER;
360 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state) {
361 if (g_recorder->audio_h) {
362 ret = audio_in_unprepare(g_recorder->audio_h);
363 if (AUDIO_IO_ERROR_NONE != ret) {
364 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to unprepare audioin : %d", ret);
368 g_recorder_state = STTD_RECORDER_STATE_READY;
371 if (g_recorder->audio_h) {
372 ret = audio_in_destroy(g_recorder->audio_h);
373 if (AUDIO_IO_ERROR_NONE != ret) {
374 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to destroy audioin : %d", ret);
376 g_recorder->audio_h = NULL;
380 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state) {
381 g_recorder_state = STTD_RECORDER_STATE_READY;
384 bt_hid_unset_audio_data_receive_cb();
390 pthread_mutex_unlock(&sttd_audio_in_handle_mutex);
391 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Leave critical section");
396 float get_volume_decibel(char* data, int size, stte_audio_type_e type)
398 #define MAX_AMPLITUDE_MEAN_16 32768
399 #define MAX_AMPLITUDE_MEAN_08 128
406 unsigned long long square_sum = 0;
408 if (type == STTE_AUDIO_TYPE_PCM_S16_LE)
413 for (i = 0; i < size; i += (depthByte<<1)) {
414 if (depthByte == 2) {
416 memcpy(&pcm16, data + i, sizeof(short));
417 square_sum += pcm16 * pcm16;
420 memcpy(&pcm8, data + i, sizeof(char));
421 square_sum += pcm8 * pcm8;
429 rms = sqrt((double)square_sum/count);
432 db = 20 * log10(rms/MAX_AMPLITUDE_MEAN_16);
434 db = 20 * log10(rms/MAX_AMPLITUDE_MEAN_08);
440 Eina_Bool __read_audio_func(void *data)
443 static char g_buffer[BUFFER_LENGTH];
445 /* Check engine id is valid */
446 if (NULL == g_recorder) {
450 if (STTD_RECORDER_STATE_READY == g_recorder_state) {
451 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Exit audio reading func");
455 read_byte = audio_in_read(g_recorder->audio_h, g_buffer, BUFFER_LENGTH);
457 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Fail to read audio : %d", read_byte);
458 g_recorder_state = STTD_RECORDER_STATE_READY;
462 if (0 != g_audio_cb(g_buffer, read_byte)) {
463 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Fail audio callback");
464 sttd_recorder_stop();
468 if (NULL == g_recorder)
469 SLOG(LOG_WARN, TAG_STTD, "[Recorder] g_recorder is NULL. It is already stopped and destroyed.");
471 if (0 == g_buffer_count % 30 && NULL != g_recorder) {
472 float vol_db = get_volume_decibel(g_buffer, BUFFER_LENGTH, g_recorder->audio_type);
473 if (0 != sttdc_send_set_volume(g_recorder->uid, vol_db)) {
474 SLOG(LOG_ERROR, TAG_STTD, "[Recorder] Fail to send recording volume(%f)", vol_db);
479 if (0 == g_buffer_count % 50) {
480 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, read_byte);
482 if (100000 == g_buffer_count) {
490 /* write pcm buffer */
491 fwrite(g_buffer, 1, BUFFER_LENGTH, g_pFile);
498 int sttd_recorder_start(int uid)
500 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state)
505 /* Check engine id is valid */
506 if (NULL == g_recorder) {
507 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
508 return STTD_ERROR_INVALID_PARAMETER;
511 ret = audio_in_set_sound_stream_info(g_recorder->audio_h, g_stream_info_h);
512 if (AUDIO_IO_ERROR_NONE != ret) {
513 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to set stream info");
516 ret = audio_in_prepare(g_recorder->audio_h);
517 if (AUDIO_IO_ERROR_NONE != ret) {
518 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to start audio : %d", ret);
519 return STTD_ERROR_RECORDER_BUSY;
522 /* Add ecore timer to read audio data */
523 if (NULL != g_audio_timer) {
524 ecore_timer_del(g_audio_timer);
525 g_audio_timer = NULL;
528 g_audio_timer = ecore_timer_add(0, __read_audio_func, NULL);
529 if (NULL == g_audio_timer) {
530 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Fail to create timer of read_audio");
531 return STTD_ERROR_OPERATION_FAILED;
535 g_recorder_state = STTD_RECORDER_STATE_RECORDING;
536 g_recorder->uid = uid;
544 snprintf(g_temp_file_name, sizeof(g_temp_file_name), "/tmp/stt_temp_%d_%d", getpid(), g_count);
545 ret = access(g_temp_file_name, 0);
548 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] File is already exist");
549 if (0 == remove(g_temp_file_name)) {
550 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Remove file");
560 SECURE_SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Temp file name=[%s]", g_temp_file_name);
563 g_pFile = fopen(g_temp_file_name, "wb+x");
565 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] File not found!");
573 int sttd_recorder_stop()
575 if (STTD_RECORDER_STATE_READY == g_recorder_state)
578 // critical section required because this function can be called from stt engine thread context
579 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Enter critical section");
580 pthread_mutex_lock(&sttd_audio_in_handle_mutex);
582 /* Check engine id is valid */
583 if (NULL == g_recorder) {
584 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
585 pthread_mutex_unlock(&sttd_audio_in_handle_mutex);
586 return STTD_ERROR_INVALID_PARAMETER;
589 g_recorder->uid = -1;
593 if (NULL != g_audio_timer) {
594 ecore_timer_del(g_audio_timer);
595 g_audio_timer = NULL;
598 ret = audio_in_unprepare(g_recorder->audio_h);
599 if (AUDIO_IO_ERROR_NONE != ret) {
600 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to unprepare audioin : %d", ret);
604 bool stopped = false;
605 while (5 > bt_retry) {
606 ret = bt_hid_rc_stop_sending_voice(NULL);
607 if (BT_ERROR_NONE == ret) {
608 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder] Stop bt audio");
611 } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
612 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail bt_hid_rc_stop_sending_voice()");
616 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] bt_hid_rc_stop_sending_voice(): ret(%d)", ret);
620 if (false == stopped) {
621 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to stop bt audio(%d)", ret);
622 // return STTD_ERROR_OPERATION_FAILED;
626 g_recorder_state = STTD_RECORDER_STATE_READY;
631 pthread_mutex_unlock(&sttd_audio_in_handle_mutex);
632 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Leave critical section");
637 int sttd_recorder_start_file(int uid, const char *filepath)
639 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state)
642 /* Check engine id is valid */
643 if (NULL == g_recorder) {
644 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
645 return STTD_ERROR_INVALID_PARAMETER;
647 g_recorder_state = STTD_RECORDER_STATE_RECORDING;
648 g_recorder->uid = uid;
651 int totalReadBytes = 0;
653 FILE *infile = fopen(filepath, "rb");
656 if (infile != NULL) {
657 while (!feof(infile)) {
658 static char pcm_buff[BUFFER_LENGTH];
659 int read_byte = fread(pcm_buff, 1, BUFFER_LENGTH, infile);
660 totalReadBytes += read_byte;
661 if (0 != read_byte) {
662 if (0 != g_audio_cb(pcm_buff, read_byte)) {
663 SLOG(LOG_ERROR, TAG_STTD, "[Recorder ERROR] Fail to call audio callback");
668 float vol_db = get_volume_decibel(pcm_buff, BUFFER_LENGTH, g_recorder->audio_type);
669 if (0 != sttdc_send_set_volume(g_recorder->uid, vol_db)) {
670 SLOG(LOG_ERROR, TAG_STTD, "[Recorder] Fail to send recording volume(%f)", vol_db);
676 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder][%d] Recording... : read_size(%d)", cnt, read_byte);
683 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder][%d] total bytes(%d)", cnt, totalReadBytes);
687 int sttd_recorder_stop_file()
689 if (STTD_RECORDER_STATE_READY == g_recorder_state)
692 /* Check engine id is valid */
693 if (NULL == g_recorder) {
694 SLOG(LOG_WARN, TAG_STTD, "[Recorder WARNING] Engine id is not valid");
695 return STTD_ERROR_INVALID_PARAMETER;
698 g_recorder->uid = -1;
699 g_recorder_state = STTD_RECORDER_STATE_READY;
704 int sttd_recorder_clear()
707 SLOG(LOG_DEBUG, TAG_STTD, "[Recorder DEBUG] Clear recorder on TV");
709 if (STTD_RECORDER_STATE_RECORDING == g_recorder_state) {
710 sttd_recorder_stop();
712 if (STTD_RECORDER_STATE_READY == g_recorder_state) {
713 sttd_recorder_destroy();
717 return STTD_ERROR_NONE;