2 * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the License);
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an AS IS BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 using System.Runtime.InteropServices;
19 using static Interop.SttEngine;
21 namespace Tizen.Uix.SttEngine
24 /// Enumeration for audio type.
29 /// Signed 16bit audio type, Little endian
33 /// Unsigned 8bit audio type
39 /// Enumeration for result.
41 public enum ResultEvent
44 /// Event when either the full matched or the final result is delivered
48 /// Event when the partial matched result is delivered
52 /// Event when the recognition has failed
58 /// Enumeration for result time.
63 /// Event when the token is beginning type
67 /// Event when the token is middle type
71 /// Event when the token is end type
77 /// Enumeration for speech status.
79 public enum SpeechStatus
82 /// Beginning point of speech is detected
84 BeginningPointDetected = 0,
86 /// End point of speech is detected
92 /// Enumeration representing the result message
94 public enum ResultMessage
101 /// Recognition failed because the speech started too soon.
105 /// Recognition failed because the speech is too short.
109 /// Recognition failed because the speech is too long.
113 /// Recognition failed because the speech is too quiet to listen.
117 /// Recognition failed because the speech is too loud to listen.
121 /// Recognition failed because the speech is too fast to listen.
128 /// Enum for Error values that can occur
133 /// Successful, No error
135 None = ErrorCode.None,
139 OutOfMemory = ErrorCode.OutOfMemory,
143 IoError = ErrorCode.IoError,
145 /// Invalid parameter
147 InvalidParameter = ErrorCode.InvalidParameter,
149 /// Network down(Out of network)
151 NetworkDown = ErrorCode.NetworkDown,
155 InvalidState = ErrorCode.InvalidState,
159 InvalidLanguage = ErrorCode.InvalidLanguage,
163 OperationFailed = ErrorCode.OperationFailed,
165 /// Not supported feature of current engine
167 NotSupportedFeature = ErrorCode.NotSupportedFeature,
171 NotSupported = ErrorCode.NotSupported,
173 /// Permission denied
175 PermissionDenied = ErrorCode.PermissionDenied,
177 /// Recording timed out
179 RecordingTimedOut = ErrorCode.RecordingTimedOut
183 /// This Class represents the Stt Engine which has to be inherited to make the engine.
185 public abstract class Engine
187 private CallbackStructGCHandle _callbackStructGCHandle = new CallbackStructGCHandle();
188 private PrivateDataSetCb _privateDataSetCb;
189 private Action<string> _privateDatacallback;
190 private PrivateDataRequestedCb _privateDataRequestedCb;
191 private OutAction<string> _privateDataRequestedCallback;
192 private static Engine _engine;
193 private IntPtr _structIntPtrHandle;
196 /// An Action with 2 Input Parameter returning a Error
198 /// <typeparam name="T">Generic Type for Parameter 1</typeparam>
199 /// <param name="a">The Input Parameter 1</param>
200 /// <param name="b">The Input Parameter 2</param>
201 /// <returns>Error Value</returns>
202 public delegate Error Action<T>(T a, T b);
205 /// An Action with 2 Out Parameter returning a Error
207 /// <typeparam name="T">Generic Type for Parameter 1</typeparam>
208 /// <param name="a">The Input Parameter 1</param>
209 /// <param name="b">The Input Parameter 2</param>
210 /// <returns>Error Value</returns>
211 public delegate Error OutAction<T>(T a, out T b);
214 /// Called when Stt engine provides the time stamp of result to the engine service user.
215 /// This callback function is implemented by the engine service user. Therefore, the engine developer does NOT have to implement this callback function.
217 /// <param name="index">The result index</param>
218 /// <param name="resultEvent">The token event</param>
219 /// <param name="text">The result text</param>
220 /// <param name="startTime">The time started speaking the result text</param>
221 /// <param name="endTime">The time finished speaking the result text</param>
222 /// <param mane="userData">The User data</param>
223 /// <returns>true to continue with the next iteration of the loop, false to break out of the loop</returns>
224 /// <precondition>SendResult() should be called.</precondition>
225 public delegate bool ResultTime(int index, TimeEvent resultEvent, string text, long startTime, long endTime, IntPtr userData);
228 /// Called when Stt engine informs the engine service user about whole supported language list.
229 /// This callback function is implemented by the engine service user. Therefore, the engine developer does NOT have to implement this callback function.
231 /// <param name="language">The language is specified as an ISO 3166 alpha-2 two letter country-code
232 /// followed by ISO 639-1 for the two-letter language code
233 /// for example, "ko_KR" for Korean, "en_US" for American English</param>
234 /// <param mane="userData">The User data</param>
235 /// <returns>true to continue with the next iteration of the loop, false to break out of the loop</returns>
236 /// <precondition>ForEachSupportedLanguages() should be called</precondition>
237 public delegate bool SupportedLanguages(string language, IntPtr userData);
240 /// Called when the engine service user requests the basic information of Stt engine.
243 /// In order to upload the engine at Tizen Appstore, both a service app and a ui app are necessary. Therefore, engineSetting must be transferred to the engine service user.
245 /// <param name="engineUuid">UUID of engine</param>
246 /// <param name="engineName">Name of engine</param>
247 /// <param name="engineSetting">The engine setting application(ui app)'s app ID</param>
248 /// <param name="useNetwork">A variable for checking whether the network is used or not</param>
250 /// Following Error Codes can be returned
252 /// 2. OperationFailed
253 /// 3. InvalidParameter
255 public abstract Error GetInformation(out string engineUuid, out string engineName, out string engineSetting, out bool useNetwork);
258 /// Called when the engine service user initializes Stt engine.
259 /// This callback function is called by the engine service user to request for Stt engine to be started.
262 /// Following Error Codes can be returned
264 /// 2. InvalidParameter
266 /// 4. OperationFailed
268 public abstract Error Initialize();
271 /// Called when the engine service user deinitializes Stt engine.
272 /// This callback function is called by the engine service user to request for Stt engine to be deinitialized.
275 /// Following Error Codes can be returned
279 public abstract Error Deinitialize();
282 /// Called when the engine service user gets the whole supported language list.
285 /// In this function, the engine service user's callback function 'SupportedLanguages' is invoked repeatedly for getting all supported languages
286 /// and user_data must be transferred to 'SupportedLanguages'. If 'SupportedLanguages' returns false, it should be stopped to call 'SupportedLanguages'.
288 /// <param name="callback">The callback function</param>
289 /// <param name="userData">The user data which must be passed to the callback delegate 'SupportedLanguages'</param>
291 /// Following Error Codes can be returned
294 /// 3. InvalidParameter
297 /// This callback function invokes SupportedLanguages repeatedly for getting supported languages.
299 public abstract Error ForEachSupportedLanguages(SupportedLanguages callback, IntPtr userData);
302 /// Called when the engine service user checks whether the corresponding language is valid or not in Stt engine.
304 /// <param name="language">The language is specified as an ISO 3166 alpha-2 two letter country-code followed by ISO 639-1 for the two-letter language code
305 /// For example, "ko_KR" for Korean, "en_US" for American English</param>
306 /// <param name="isValid">A variable for checking whether the corresponding language is valid or not. true to be valid, false to be invalid</param>
308 /// Following Error Codes can be returned
310 /// 2. InvalidParameter
312 public abstract Error IsValidLanguage(string language, out bool isValid);
315 /// Called when the engine service user checks whether Stt engine supports silence detection.
317 /// <returns>true to support silence detection, false not to support silence detection</returns>
318 public abstract bool SupportSilenceDetection();
321 /// Called when the engine service user checks whether Stt engine supports the corresponding recognition type.
323 /// <param name="type">The type for recognition, "stt.recognition.type.FREE" or "stt.recognition.type.FREE.PARTIAL"</param>
324 /// <param name="isSupported">A variable for checking whether Stt engine supports the corresponding recognition type.
325 /// true to support recognition type, false not to support recognition type</param>
327 /// Following Error Codes can be returned
329 /// 2. InvalidParameter
331 public abstract Error SupportRecognitionType(string type, out bool isSupported);
334 /// Called when the engine service user gets the proper recording format of Stt engine.
335 /// The recording format is used for creating the recorder.
337 /// <param name="types">The format used by the recorder</param>
338 /// <param name="rate">The sample rate used by the recorder</param>
339 /// <param name="channels">The number of channels used by the recorder</param>
341 /// Following Error Codes can be returned
345 public abstract Error GetRecordingFormat(out AudioType types, out int rate, out int channels);
348 /// Called when the engine service user sets the silence detection.
349 /// If the engine service user sets this option as 'TRUE', Stt engine will detect the silence (EPD) and send the callback event about it.
351 /// <param name="isSet">A variable for setting the silence detection. true to detect the silence, false not to detect the silence</param>
353 /// Following Error Codes can be returned
356 /// 3. NotSupportedFeature
358 public abstract Error SetSilenceDetection(bool isSet);
361 /// Called when the engine service user requests for Stt engine to check whether the application agreed the usage of Stt engine.
362 /// This callback function is called when the engine service user requests for Stt engine to check the application's agreement about using the engine.
363 /// According to the need, the engine developer can provide some user interfaces to check the agreement.
365 /// <param name="appid">The Application ID</param>
366 /// <param name="isAgreed">A variable for checking whether the application agreed to use Stt engine or not. true to agree, false to disagree</param>
368 /// Following Error Codes can be returned
371 /// 3. NotSupportedFeature
373 public abstract Error CheckAppAgreed(string appid, out bool isAgreed);
376 /// Called when the engine service user checks whether Stt engine needs the application's credential.
378 /// <returns>true if Stt engine needs the application's credential, otherwise false </returns>
379 public abstract bool NeedAppCredential();
382 /// Called when the engine service user gets the result time information(stamp).
385 /// In this function, the engine service user's callback delegate 'ResultTime' is invoked repeatedly for sending the time information to the engine service user
386 /// and user_data must be transferred to 'ResultTime'. If 'ResultTime' returns false, it should be stopped to call 'ResultTime'.
387 /// timeInfo is transferred from SendResult. The type of timeInfo is up to the Stt engine developer.
389 /// <param name="timeInfo">The time information</param>
390 /// <param name="callback">The callback function</param>
391 /// <param name="userData">The user data which must be passed to the callback function ResultTime</param>
393 /// Following Error Codes can be returned
396 /// 3. InvalidParameter
399 /// SendResult will invoke this function
402 /// This function invokes ResultTime repeatedly for getting result time information.
404 public abstract Error ForEachResultTime(IntPtr timeInfo, ResultTime callback, IntPtr userData);
407 /// Called when the engine service user starts to recognize the recording data.
408 /// In this callback function, Stt engine must transfer the recognition result and userData to the engine service user using SendResult().
409 /// Also, if Stt engine needs the application's credential, it sets the credential granted to the application.
411 /// <param name="language">The language is specified as an ISO 3166 alpha-2 two letter country-code followed by ISO 639-1 for the two-letter language code
412 /// For example, "ko_KR" for Korean, "en_US" for American English</param>
413 /// <param name="type">The recognition type, "stt.recognition.type.FREE" or "stt.recognition.type.FREE.PARTIAL"</param>
414 /// <param name="appid">The Application ID</param>
415 /// <param name="credential">The credential granted to the application</param>
416 /// <param name="userData">The user data to be passed to the callback function</param>
418 /// Following Error Codes can be returned
421 /// 3. InvalidParameter
422 /// 4. InvalidLanguage
423 /// 5. OperationFailed
427 /// The engine is not in recognition processing.
429 public abstract Error Start(string language, string type, string appid, string credential, IntPtr userData);
432 /// Called when the engine service user sets and sends the recording data for speech recognition.
433 /// This callback function is called by the engine service user to send the recording data to Stt engine.The engine receives the recording data and uses for speech recognition.
434 /// this function should be returned immediately after recording data copy.
436 /// <param name="data">The recording data</param>
437 /// <param name="length">The length of recording data</param>
439 /// Following Error Codes can be returned
442 /// 3. InvalidParameter
443 /// 4. OperationFailed
446 /// Start should succeed</percondition>
448 /// If the engine supports partial result, SendResult() should be invoked.</postcondition>
449 public abstract Error SetRecordingData(string data, uint length);
452 /// Called when the engine service user stops to recognize the recording data.
453 /// This callback function is called by the engine service user to stop recording and to get the recognition result.
455 /// <returns>Following Error Codes can be returned
458 /// 3. OperationFailed
462 /// Start should succeed</percondition>
464 /// After processing of the engine, , SendResult() should be invoked.</postcondition>
465 public abstract Error Stop();
468 /// Called when the engine service user cancels to recognize the recording data.
469 /// This callback function is called by the engine service user to cancel to recognize the recording data.Also, when starting the recorder is failed, this function is called.
472 /// Following Error Codes can be returned
476 /// <precondition>Stt engine is in recognition processing or recording.</precondition>
477 public abstract Error Cancel();
480 /// Public Constructor
488 /// Main function for Speech-To-Text (STT) engine.
489 /// This function is the main function for operating Stt engine.
492 /// http://tizen.org/privilege/recorder
495 /// ServiceAppMain should be used for working the engine after this function.
497 /// <param name="argc">The Number of Arguments</param>
498 /// <param name="argv">The Arguments Array</param>
499 /// <exception cref="ArgumentException">Thrown in case of Invalid Parameter</exception>
500 /// <exception cref="UnauthorizedAccessException">Thrown in case of Permission denied</exception>
501 /// <exception cref="NotSupportedException">Thrown in case of Not supported</exception>
502 /// <exception cref="InvalidOperationException">thrown in case of Operation failure</exception>
503 public void EngineMain(int argc, string[] argv)
505 _callbackStructGCHandle.CallbackStruct.version = 1;
506 _callbackStructGCHandle.CallbackStruct.getInfo = _getInfoCb;
507 _callbackStructGCHandle.CallbackStruct.initialize = Initialize;
508 _callbackStructGCHandle.CallbackStruct.deinitialize = _deinitializeCb;
509 _callbackStructGCHandle.CallbackStruct.supportedLanaguge = ForEachSupportedLanguages;
510 _callbackStructGCHandle.CallbackStruct.validLanaguage = _isValidLanguageCb;
511 _callbackStructGCHandle.CallbackStruct.silence = SupportSilenceDetection;
512 _callbackStructGCHandle.CallbackStruct.recognitionType = SupportRecognitionType;
513 _callbackStructGCHandle.CallbackStruct.recordingFormat = GetRecordingFormat;
514 _callbackStructGCHandle.CallbackStruct.resultTime = ForEachResultTime;
515 _callbackStructGCHandle.CallbackStruct.silenceDetection = SetSilenceDetection;
516 _callbackStructGCHandle.CallbackStruct.start = _startCb;
517 _callbackStructGCHandle.CallbackStruct.recordingData = SetRecordingData;
518 _callbackStructGCHandle.CallbackStruct.stop = Stop;
519 _callbackStructGCHandle.CallbackStruct.cancel = Cancel;
520 _callbackStructGCHandle.CallbackStruct.checkAppAgreed = CheckAppAgreed;
521 _callbackStructGCHandle.CallbackStruct.needAppCredential = NeedAppCredential;
522 _structIntPtrHandle = Marshal.AllocHGlobal(Marshal.SizeOf(_callbackStructGCHandle.CallbackStruct));
523 Marshal.StructureToPtr<RequestCallbackStruct>(_callbackStructGCHandle.CallbackStruct, _structIntPtrHandle, false);
524 Error error = STTEMain(argc, argv, _structIntPtrHandle);
525 if (error != Error.None)
527 Log.Error(LogTag, "STTEMain Failed with error " + error);
528 throw ExceptionFactory.CreateException((ErrorCode)error);
531 Log.Info(LogTag, "After STTEMain");
535 /// Sends the recognition result to the engine service user.
538 /// This API is used in SetRecordingData() and Stop(), when Stt engine sends the recognition result to the engine service user.
539 /// This function is called in the following situations; 1) after Stop() is called, 2) the end point of speech is detected from recording, or 3) partial result is occurred.
540 /// The recognition result must be transferred to the engine service user through this function. Also, timeInfo must be transferred to ForEachResultTime().
541 /// The type of timeInfo is up to the Stt engine developer.
543 /// <param name="resultEvent">The result event</param>
544 /// <param name="type">The recognition type, "stt.recognition.type.FREE" or "stt.recognition.type.FREE.PARTIAL"</param>
545 /// <param name="result">Result texts</param>
546 /// <param name="resultCount">Result text count</param>
547 /// <param name="msg">Engine message</param>
548 /// <param name="timeInfo">The time information</param>
549 /// <exception cref="ArgumentException">Thrown in case of Invalid Parameter</exception>
550 /// <exception cref="UnauthorizedAccessException">Thrown in case of Permission denied</exception>
551 /// <exception cref="NotSupportedException">Thrown in case of Not supported</exception>
552 /// <exception cref="InvalidOperationException">thrown in case of Operation failure</exception>
554 /// EngineMain function should be invoked before this function is called. Stop will invoke this function.
557 /// This function invokes ForEachResultTime
559 public void SendResult(ResultEvent resultEvent, string type, string[] result, int resultCount, ResultMessage msg, IntPtr timeInfo)
561 if ((result != null) && (result.Length != 0))
564 string message = "stt.result.message.none";
567 case ResultMessage.None:
568 message = "stt.result.message.none";
571 case ResultMessage.TooFast:
572 message = "stt.result.message.error.too.fast";
575 case ResultMessage.TooLong:
576 message = "stt.result.message.error.too.long";
579 case ResultMessage.TooLoud:
580 message = "stt.result.message.error.too.loud";
583 case ResultMessage.TooQuiet:
584 message = "stt.result.message.error.too.quiet";
587 case ResultMessage.TooShort:
588 message = "stt.result.message.error.too.short";
591 case ResultMessage.TooSoon:
592 message = "stt.result.message.error.too.soon";
596 Error error = STTESendResult(resultEvent, type, result, resultCount, message, timeInfo, IntPtr.Zero);
597 if (error != Error.None)
599 Log.Error(LogTag, "STTESendResult Failed with error " + error);
600 throw ExceptionFactory.CreateException((ErrorCode)error);
606 throw new ArgumentNullException("result", "is null or empty");
611 /// Sends the error to the engine service user.
613 /// <param name="error">The Error Reason</param>
614 /// <param name="msg">The error message</param>
615 /// <exception cref="ArgumentException">Thrown in case of Invalid Parameter</exception>
616 /// <exception cref="UnauthorizedAccessException">Thrown in case of Permission denied</exception>
617 /// <exception cref="NotSupportedException">Thrown in case of Not supported</exception>
618 /// <exception cref="InvalidOperationException">thrown in case of Operation failure</exception>
620 /// Main function should be invoked before this function is called.
622 public void SendError(Error error, string msg)
624 Error err = STTESendError(error, msg);
625 if (err != Error.None)
627 Log.Error(LogTag, "SendError Failed with error " + err);
628 throw ExceptionFactory.CreateException((ErrorCode)error);
634 /// Sends the speech status to the engine service user when Stt engine notifies the change of the speech status.
637 /// This API is invoked when Stt engine wants to notify the change of the speech status anytime. NOTE that this API can be invoked for recognizing the speech.
639 /// <param name="status">SpeechStatus</param>
640 /// <exception cref="ArgumentException">Thrown in case of Invalid Parameter</exception>
641 /// <exception cref="UnauthorizedAccessException">Thrown in case of Permission denied</exception>
642 /// <exception cref="NotSupportedException">Thrown in case of Not supported</exception>
643 /// <exception cref="InvalidOperationException">thrown in case of Operation failure</exception>
645 /// Main function should be invoked before this function is called. Start() and SetRecordingData() will invoke this function.
647 public void SendSpeechStatus(SpeechStatus status)
649 Error error = STTESendSpeechStatus(status, IntPtr.Zero);
650 if (error != Error.None)
652 Log.Error(LogTag, "SendSpeechStatus Failed with error " + error);
653 throw ExceptionFactory.CreateException((ErrorCode)error);
659 /// Sets a callback function for setting the private data.
662 /// http://tizen.org/privilege/recorder
664 /// <param name="callback">
665 /// Called when Stt engine receives the private data from the engine service user.
666 /// This callback function is called when the engine service user sends the private data to Stt engine.
668 /// a = Key -- The key field of private data
669 /// b = data -- The data field of private data
670 /// Following Error Codes can be returned
672 /// 2. InvalidParameter
673 /// 3. OperationFailed
675 /// <exception cref="ArgumentException">Thrown in case of Invalid Parameter</exception>
676 /// <exception cref="UnauthorizedAccessException">Thrown in case of Permission denied</exception>
677 /// <exception cref="NotSupportedException">Thrown in case of Not supported</exception>
678 /// <exception cref="InvalidOperationException">thrown in case of Operation failure</exception>
680 /// Main function should be invoked before this function is called.
682 public void SetPrivateDataSetDelegate(Action<string> callback)
684 _privateDatacallback = callback;
685 _privateDataSetCb = (string key, string data) =>
687 return _privateDatacallback.Invoke(key, data);
689 Error error = STTESetPrivateDataSetCb(_privateDataSetCb);
690 if (error != Error.None)
692 Log.Error(LogTag, "SetPrivateDataSetDelegate Failed with error " + error);
693 throw ExceptionFactory.CreateException((ErrorCode)error);
699 /// Sets a callback delegate for requesting the private data.
702 /// http://tizen.org/privilege/recorder
704 /// <param name="callback">callback function
705 /// Called when Stt engine provides the engine service user with the private data.
706 /// This callback function is called when the engine service user gets the private data from Stt engine.
708 /// a = Key -- The key field of private data
709 /// b = data -- The data field of private data
710 /// Following Error Codes can be returned
712 /// 2. InvalidParameter
713 /// 3. OperationFailed
715 /// <exception cref="ArgumentException">Thrown in case of Invalid Parameter</exception>
716 /// <exception cref="UnauthorizedAccessException">Thrown in case of Permission denied</exception>
717 /// <exception cref="NotSupportedException">Thrown in case of Not supported</exception>
718 /// <exception cref="InvalidOperationException">thrown in case of Operation failure</exception>
720 /// Main function should be invoked before this function is called.
722 public void SetPrivateDataRequestedDelegate(OutAction<string> callback)
724 _privateDataRequestedCallback = callback;
725 _privateDataRequestedCb = (string key, out string data) =>
727 return _privateDataRequestedCallback.Invoke(key, out data);
729 Error error = STTESetPrivateDataRequestedCb(_privateDataRequestedCb);
730 if (error != Error.None)
732 Log.Error(LogTag, "SetPrivateDataRequestedDelegate Failed with error " + error);
733 throw ExceptionFactory.CreateException((ErrorCode)error);
738 private StartCb _startCb = (IntPtr language, IntPtr type, IntPtr appid, IntPtr credential, IntPtr userData) =>
744 if (language != null)
745 lan = Marshal.PtrToStringAnsi(language);
747 typ = Marshal.PtrToStringAnsi(type);
749 apid = Marshal.PtrToStringAnsi(appid);
750 if (credential != null)
751 cre = Marshal.PtrToStringAnsi(credential);
752 return _engine.Start(lan, typ, apid, cre, IntPtr.Zero);
755 private IsValidLanguageCb _isValidLanguageCb = (IntPtr langauge, IntPtr isValid) =>
757 string langaugeStr = Marshal.PtrToStringAnsi(langauge);
759 Error err = _engine.IsValidLanguage(langaugeStr, out valid);
762 Marshal.WriteByte(isValid, 0, 1);
766 Marshal.WriteByte(isValid, 0, 0);
771 private GetInfoCb _getInfoCb = (out IntPtr engineUuid, out IntPtr engineName, out IntPtr engineSetting, out IntPtr useNetwork) =>
777 Error err = _engine.GetInformation(out uuid, out name, out setting, out network);
778 int size = Marshal.SizeOf<int>();
779 IntPtr pBool = Marshal.AllocHGlobal(size);
782 Marshal.WriteInt32(pBool, 0, 1);
786 Marshal.WriteInt32(pBool, 0, 0);
788 engineUuid = Marshal.StringToHGlobalAnsi(uuid);
789 engineName = Marshal.StringToHGlobalAnsi(name);
790 engineSetting = Marshal.StringToHGlobalAnsi(setting);
795 private DeinitializeCb _deinitializeCb = () =>
797 Marshal.FreeHGlobal(_engine._structIntPtrHandle);
798 return _engine.Deinitialize();