/* * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the License); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an AS IS BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using System.Runtime.InteropServices; using static Interop.SttEngine; namespace Tizen.Uix.SttEngine { /// /// Enumeration for audio type. /// public enum AudioType { /// /// Signed 16bit audio type, Little endian /// PcmS16Le = 0, /// /// Unsigned 8bit audio type /// PcmU8 }; /// /// Enumeration for result. /// public enum ResultEvent { /// /// Event when either the full matched or the final result is delivered /// FinalResult = 0, /// /// Event when the partial matched result is delivered /// PartialResult, /// /// Event when the recognition has failed /// Error }; /// /// Enumeration for result time. /// public enum TimeEvent { /// /// Event when the token is beginning type /// Beginning = 0, /// /// Event when the token is middle type /// Middle = 1, /// /// Event when the token is end type /// End = 2 }; /// /// Enumeration for speech status. /// public enum SpeechStatus { /// /// Beginning point of speech is detected /// BeginningPointDetected = 0, /// /// End point of speech is detected /// EndPointDetected }; /// /// Enumeration representing the result message /// public enum ResultMessage { /// /// No Error /// None, /// /// Recognition failed because the speech started too soon. /// TooSoon, /// /// Recognition failed because the speech is too short. /// TooShort, /// /// Recognition failed because the speech is too long. /// TooLong, /// /// Recognition failed because the speech is too quiet to listen. /// TooQuiet, /// /// Recognition failed because the speech is too loud to listen. /// TooLoud, /// /// Recognition failed because the speech is too fast to listen. /// TooFast }; /// /// Enum for Error values that can occur /// public enum Error { /// /// Successful, No error /// None = ErrorCode.None, /// /// Out of Memory /// OutOfMemory = ErrorCode.OutOfMemory, /// /// I/O error /// IoError = ErrorCode.IoError, /// /// Invalid parameter /// InvalidParameter = ErrorCode.InvalidParameter, /// /// Network down(Out of network) /// NetworkDown = ErrorCode.NetworkDown, /// /// Invalid state /// InvalidState = ErrorCode.InvalidState, /// /// Invalid language /// InvalidLanguage = ErrorCode.InvalidLanguage, /// /// Operation failed /// OperationFailed = ErrorCode.OperationFailed, /// /// Not supported feature of current engine /// NotSupportedFeature = ErrorCode.NotSupportedFeature, /// /// NOT supported /// NotSupported = ErrorCode.NotSupported, /// /// Permission denied /// PermissionDenied = ErrorCode.PermissionDenied, /// /// Recording timed out /// RecordingTimedOut = ErrorCode.RecordingTimedOut }; /// /// This Class represents the Stt Engine which has to be inherited to make the engine. /// public abstract class Engine { private CallbackStructGCHandle _callbackStructGCHandle = new CallbackStructGCHandle(); private PrivateDataSetCb _privateDataSetCb; private Action _privateDatacallback; private PrivateDataRequestedCb _privateDataRequestedCb; private OutAction _privateDataRequestedCallback; private static Engine _engine; private IntPtr _structIntPtrHandle; /// /// An Action with 2 Input Parameter returning a Error /// /// Generic Type for Parameter 1 /// The Input Parameter 1 /// The Input Parameter 2 /// Error Value public delegate Error Action(T a, T b); /// /// An Action with 2 Out Parameter returning a Error /// /// Generic Type for Parameter 1 /// The Input Parameter 1 /// The Input Parameter 2 /// Error Value public delegate Error OutAction(T a, out T b); /// /// Called when Stt engine provides the time stamp of result to the engine service user. /// This callback function is implemented by the engine service user. Therefore, the engine developer does NOT have to implement this callback function. /// /// The result index /// The token event /// The result text /// The time started speaking the result text /// The time finished speaking the result text /// The User data /// true to continue with the next iteration of the loop, false to break out of the loop /// SendResult() should be called. public delegate bool ResultTime(int index, TimeEvent resultEvent, string text, long startTime, long endTime, IntPtr userData); /// /// Called when Stt engine informs the engine service user about whole supported language list. /// This callback function is implemented by the engine service user. Therefore, the engine developer does NOT have to implement this callback function. /// /// The language is specified as an ISO 3166 alpha-2 two letter country-code /// followed by ISO 639-1 for the two-letter language code /// for example, "ko_KR" for Korean, "en_US" for American English /// The User data /// true to continue with the next iteration of the loop, false to break out of the loop /// ForEachSupportedLanguages() should be called public delegate bool SupportedLanguages(string language, IntPtr userData); /// /// Called when the engine service user requests the basic information of Stt engine. /// /// /// In order to upload the engine at Tizen Appstore, both a service app and a ui app are necessary. Therefore, engineSetting must be transferred to the engine service user. /// /// UUID of engine /// Name of engine /// The engine setting application(ui app)'s app ID /// A variable for checking whether the network is used or not /// /// Following Error Codes can be returned /// 1. None /// 2. OperationFailed /// 3. InvalidParameter /// public abstract Error GetInformation(out string engineUuid, out string engineName, out string engineSetting, out bool useNetwork); /// /// Called when the engine service user initializes Stt engine. /// This callback function is called by the engine service user to request for Stt engine to be started. /// /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidParameter /// 3. InvalidState /// 4. OperationFailed /// public abstract Error Initialize(); /// /// Called when the engine service user deinitializes Stt engine. /// This callback function is called by the engine service user to request for Stt engine to be deinitialized. /// /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// public abstract Error Deinitialize(); /// /// Called when the engine service user gets the whole supported language list. /// /// /// In this function, the engine service user's callback function 'SupportedLanguages' is invoked repeatedly for getting all supported languages /// and user_data must be transferred to 'SupportedLanguages'. If 'SupportedLanguages' returns false, it should be stopped to call 'SupportedLanguages'. /// /// The callback function /// The user data which must be passed to the callback delegate 'SupportedLanguages' /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. InvalidParameter /// /// /// This callback function invokes SupportedLanguages repeatedly for getting supported languages. /// public abstract Error ForEachSupportedLanguages(SupportedLanguages callback, IntPtr userData); /// /// Called when the engine service user checks whether the corresponding language is valid or not in Stt engine. /// /// The language is specified as an ISO 3166 alpha-2 two letter country-code followed by ISO 639-1 for the two-letter language code /// For example, "ko_KR" for Korean, "en_US" for American English /// A variable for checking whether the corresponding language is valid or not. true to be valid, false to be invalid /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidParameter /// public abstract Error IsValidLanguage(string language, out bool isValid); /// /// Called when the engine service user checks whether Stt engine supports silence detection. /// /// true to support silence detection, false not to support silence detection public abstract bool SupportSilenceDetection(); /// /// Called when the engine service user checks whether Stt engine supports the corresponding recognition type. /// /// The type for recognition, "stt.recognition.type.FREE" or "stt.recognition.type.FREE.PARTIAL" /// A variable for checking whether Stt engine supports the corresponding recognition type. /// true to support recognition type, false not to support recognition type /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidParameter /// public abstract Error SupportRecognitionType(string type, out bool isSupported); /// /// Called when the engine service user gets the proper recording format of Stt engine. /// The recording format is used for creating the recorder. /// /// The format used by the recorder /// The sample rate used by the recorder /// The number of channels used by the recorder /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// public abstract Error GetRecordingFormat(out AudioType types, out int rate, out int channels); /// /// Called when the engine service user sets the silence detection. /// If the engine service user sets this option as 'TRUE', Stt engine will detect the silence (EPD) and send the callback event about it. /// /// A variable for setting the silence detection. true to detect the silence, false not to detect the silence /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. NotSupportedFeature /// public abstract Error SetSilenceDetection(bool isSet); /// /// Called when the engine service user requests for Stt engine to check whether the application agreed the usage of Stt engine. /// This callback function is called when the engine service user requests for Stt engine to check the application's agreement about using the engine. /// According to the need, the engine developer can provide some user interfaces to check the agreement. /// /// The Application ID /// A variable for checking whether the application agreed to use Stt engine or not. true to agree, false to disagree /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. NotSupportedFeature /// public abstract Error CheckAppAgreed(string appid, out bool isAgreed); /// /// Called when the engine service user checks whether Stt engine needs the application's credential. /// /// true if Stt engine needs the application's credential, otherwise false public abstract bool NeedAppCredential(); /// /// Called when the engine service user gets the result time information(stamp). /// /// /// In this function, the engine service user's callback delegate 'ResultTime' is invoked repeatedly for sending the time information to the engine service user /// and user_data must be transferred to 'ResultTime'. If 'ResultTime' returns false, it should be stopped to call 'ResultTime'. /// timeInfo is transferred from SendResult. The type of timeInfo is up to the Stt engine developer. /// /// The time information /// The callback function /// The user data which must be passed to the callback function ResultTime /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. InvalidParameter /// /// /// SendResult will invoke this function /// /// /// This function invokes ResultTime repeatedly for getting result time information. /// public abstract Error ForEachResultTime(IntPtr timeInfo, ResultTime callback, IntPtr userData); /// /// Called when the engine service user starts to recognize the recording data. /// In this callback function, Stt engine must transfer the recognition result and userData to the engine service user using SendResult(). /// Also, if Stt engine needs the application's credential, it sets the credential granted to the application. /// /// The language is specified as an ISO 3166 alpha-2 two letter country-code followed by ISO 639-1 for the two-letter language code /// For example, "ko_KR" for Korean, "en_US" for American English /// The recognition type, "stt.recognition.type.FREE" or "stt.recognition.type.FREE.PARTIAL" /// The Application ID /// The credential granted to the application /// The user data to be passed to the callback function /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. InvalidParameter /// 4. InvalidLanguage /// 5. OperationFailed /// 6. NetworkDown /// /// /// The engine is not in recognition processing. /// public abstract Error Start(string language, string type, string appid, string credential, IntPtr userData); /// /// Called when the engine service user sets and sends the recording data for speech recognition. /// This callback function is called by the engine service user to send the recording data to Stt engine.The engine receives the recording data and uses for speech recognition. /// this function should be returned immediately after recording data copy. /// /// The recording data /// The length of recording data /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. InvalidParameter /// 4. OperationFailed /// /// /// Start should succeed /// /// If the engine supports partial result, SendResult() should be invoked. public abstract Error SetRecordingData(string data, uint length); /// /// Called when the engine service user stops to recognize the recording data. /// This callback function is called by the engine service user to stop recording and to get the recognition result. /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// 3. OperationFailed /// 4. NetworkDown /// /// /// Start should succeed /// /// After processing of the engine, , SendResult() should be invoked. public abstract Error Stop(); /// /// Called when the engine service user cancels to recognize the recording data. /// This callback function is called by the engine service user to cancel to recognize the recording data.Also, when starting the recorder is failed, this function is called. /// /// /// Following Error Codes can be returned /// 1. None /// 2. InvalidState /// /// Stt engine is in recognition processing or recording. public abstract Error Cancel(); /// /// Public Constructor /// public Engine() { _engine = this; } /// /// Main function for Speech-To-Text (STT) engine. /// This function is the main function for operating Stt engine. /// /// /// http://tizen.org/privilege/recorder /// /// /// ServiceAppMain should be used for working the engine after this function. /// /// The Number of Arguments /// The Arguments Array /// Thrown in case of Invalid Parameter /// Thrown in case of Permission denied /// Thrown in case of Not supported /// thrown in case of Operation failure public void EngineMain(int argc, string[] argv) { _callbackStructGCHandle.CallbackStruct.version = 1; _callbackStructGCHandle.CallbackStruct.getInfo = _getInfoCb; _callbackStructGCHandle.CallbackStruct.initialize = Initialize; _callbackStructGCHandle.CallbackStruct.deinitialize = _deinitializeCb; _callbackStructGCHandle.CallbackStruct.supportedLanaguge = ForEachSupportedLanguages; _callbackStructGCHandle.CallbackStruct.validLanaguage = _isValidLanguageCb; _callbackStructGCHandle.CallbackStruct.silence = SupportSilenceDetection; _callbackStructGCHandle.CallbackStruct.recognitionType = SupportRecognitionType; _callbackStructGCHandle.CallbackStruct.recordingFormat = GetRecordingFormat; _callbackStructGCHandle.CallbackStruct.resultTime = ForEachResultTime; _callbackStructGCHandle.CallbackStruct.silenceDetection = SetSilenceDetection; _callbackStructGCHandle.CallbackStruct.start = _startCb; _callbackStructGCHandle.CallbackStruct.recordingData = SetRecordingData; _callbackStructGCHandle.CallbackStruct.stop = Stop; _callbackStructGCHandle.CallbackStruct.cancel = Cancel; _callbackStructGCHandle.CallbackStruct.checkAppAgreed = CheckAppAgreed; _callbackStructGCHandle.CallbackStruct.needAppCredential = NeedAppCredential; _structIntPtrHandle = Marshal.AllocHGlobal(Marshal.SizeOf(_callbackStructGCHandle.CallbackStruct)); Marshal.StructureToPtr(_callbackStructGCHandle.CallbackStruct, _structIntPtrHandle, false); Error error = STTEMain(argc, argv, _structIntPtrHandle); if (error != Error.None) { Log.Error(LogTag, "STTEMain Failed with error " + error); throw ExceptionFactory.CreateException((ErrorCode)error); } Log.Info(LogTag, "After STTEMain"); } /// /// Sends the recognition result to the engine service user. /// /// /// This API is used in SetRecordingData() and Stop(), when Stt engine sends the recognition result to the engine service user. /// This function is called in the following situations; 1) after Stop() is called, 2) the end point of speech is detected from recording, or 3) partial result is occurred. /// The recognition result must be transferred to the engine service user through this function. Also, timeInfo must be transferred to ForEachResultTime(). /// The type of timeInfo is up to the Stt engine developer. /// /// The result event /// The recognition type, "stt.recognition.type.FREE" or "stt.recognition.type.FREE.PARTIAL" /// Result texts /// Result text count /// Engine message /// The time information /// Thrown in case of Invalid Parameter /// Thrown in case of Permission denied /// Thrown in case of Not supported /// thrown in case of Operation failure /// /// EngineMain function should be invoked before this function is called. Stop will invoke this function. /// /// /// This function invokes ForEachResultTime /// public void SendResult(ResultEvent resultEvent, string type, string[] result, int resultCount, ResultMessage msg, IntPtr timeInfo) { if ((result != null) && (result.Length != 0)) { string message = "stt.result.message.none"; switch (msg) { case ResultMessage.None: message = "stt.result.message.none"; break; case ResultMessage.TooFast: message = "stt.result.message.error.too.fast"; break; case ResultMessage.TooLong: message = "stt.result.message.error.too.long"; break; case ResultMessage.TooLoud: message = "stt.result.message.error.too.loud"; break; case ResultMessage.TooQuiet: message = "stt.result.message.error.too.quiet"; break; case ResultMessage.TooShort: message = "stt.result.message.error.too.short"; break; case ResultMessage.TooSoon: message = "stt.result.message.error.too.soon"; break; } Error error = STTESendResult(resultEvent, type, result, resultCount, message, timeInfo, IntPtr.Zero); if (error != Error.None) { Log.Error(LogTag, "STTESendResult Failed with error " + error); throw ExceptionFactory.CreateException((ErrorCode)error); } } else { throw new ArgumentNullException("result", "is null or empty"); } } /// /// Sends the error to the engine service user. /// /// The Error Reason /// The error message /// Thrown in case of Invalid Parameter /// Thrown in case of Permission denied /// Thrown in case of Not supported /// thrown in case of Operation failure /// /// Main function should be invoked before this function is called. /// public void SendError(Error error, string msg) { Error err = STTESendError(error, msg); if (err != Error.None) { Log.Error(LogTag, "SendError Failed with error " + err); throw ExceptionFactory.CreateException((ErrorCode)error); } } /// /// Sends the speech status to the engine service user when Stt engine notifies the change of the speech status. /// /// /// This API is invoked when Stt engine wants to notify the change of the speech status anytime. NOTE that this API can be invoked for recognizing the speech. /// /// SpeechStatus /// Thrown in case of Invalid Parameter /// Thrown in case of Permission denied /// Thrown in case of Not supported /// thrown in case of Operation failure /// /// Main function should be invoked before this function is called. Start() and SetRecordingData() will invoke this function. /// public void SendSpeechStatus(SpeechStatus status) { Error error = STTESendSpeechStatus(status, IntPtr.Zero); if (error != Error.None) { Log.Error(LogTag, "SendSpeechStatus Failed with error " + error); throw ExceptionFactory.CreateException((ErrorCode)error); } } /// /// Sets a callback function for setting the private data. /// /// /// http://tizen.org/privilege/recorder /// /// /// Called when Stt engine receives the private data from the engine service user. /// This callback function is called when the engine service user sends the private data to Stt engine. /// In Parameters: /// a = Key -- The key field of private data /// b = data -- The data field of private data /// Following Error Codes can be returned /// 1. None /// 2. InvalidParameter /// 3. OperationFailed /// /// Thrown in case of Invalid Parameter /// Thrown in case of Permission denied /// Thrown in case of Not supported /// thrown in case of Operation failure /// /// Main function should be invoked before this function is called. /// public void SetPrivateDataSetDelegate(Action callback) { _privateDatacallback = callback; _privateDataSetCb = (string key, string data) => { return _privateDatacallback.Invoke(key, data); }; Error error = STTESetPrivateDataSetCb(_privateDataSetCb); if (error != Error.None) { Log.Error(LogTag, "SetPrivateDataSetDelegate Failed with error " + error); throw ExceptionFactory.CreateException((ErrorCode)error); } } /// /// Sets a callback delegate for requesting the private data. /// /// /// http://tizen.org/privilege/recorder /// /// callback function /// Called when Stt engine provides the engine service user with the private data. /// This callback function is called when the engine service user gets the private data from Stt engine. /// Out Parameters: /// a = Key -- The key field of private data /// b = data -- The data field of private data /// Following Error Codes can be returned /// 1. None /// 2. InvalidParameter /// 3. OperationFailed /// /// Thrown in case of Invalid Parameter /// Thrown in case of Permission denied /// Thrown in case of Not supported /// thrown in case of Operation failure /// /// Main function should be invoked before this function is called. /// public void SetPrivateDataRequestedDelegate(OutAction callback) { _privateDataRequestedCallback = callback; _privateDataRequestedCb = (string key, out string data) => { return _privateDataRequestedCallback.Invoke(key, out data); }; Error error = STTESetPrivateDataRequestedCb(_privateDataRequestedCb); if (error != Error.None) { Log.Error(LogTag, "SetPrivateDataRequestedDelegate Failed with error " + error); throw ExceptionFactory.CreateException((ErrorCode)error); } } private StartCb _startCb = (IntPtr language, IntPtr type, IntPtr appid, IntPtr credential, IntPtr userData) => { string lan = null; string typ = null; string apid = null; string cre = null; if (language != null) lan = Marshal.PtrToStringAnsi(language); if (type != null) typ = Marshal.PtrToStringAnsi(type); if (appid != null) apid = Marshal.PtrToStringAnsi(appid); if (credential != null) cre = Marshal.PtrToStringAnsi(credential); return _engine.Start(lan, typ, apid, cre, IntPtr.Zero); }; private IsValidLanguageCb _isValidLanguageCb = (IntPtr langauge, IntPtr isValid) => { string langaugeStr = Marshal.PtrToStringAnsi(langauge); bool valid; Error err = _engine.IsValidLanguage(langaugeStr, out valid); if (valid == true) { Marshal.WriteByte(isValid, 0, 1); } else { Marshal.WriteByte(isValid, 0, 0); } return err; }; private GetInfoCb _getInfoCb = (out IntPtr engineUuid, out IntPtr engineName, out IntPtr engineSetting, out IntPtr useNetwork) => { string uuid; string name; string setting; bool network; Error err = _engine.GetInformation(out uuid, out name, out setting, out network); int size = Marshal.SizeOf(); IntPtr pBool = Marshal.AllocHGlobal(size); if (network == true) { Marshal.WriteInt32(pBool, 0, 1); } else { Marshal.WriteInt32(pBool, 0, 0); } engineUuid = Marshal.StringToHGlobalAnsi(uuid); engineName = Marshal.StringToHGlobalAnsi(name); engineSetting = Marshal.StringToHGlobalAnsi(setting); useNetwork = pBool; return err; }; private DeinitializeCb _deinitializeCb = () => { Marshal.FreeHGlobal(_engine._structIntPtrHandle); return _engine.Deinitialize(); }; } }