plugins/wakeup-manager/inc/wakeup_manager.h

   1 /*
   2  * Copyright 2018-2019 Samsung Electronics Co., Ltd
   3  *
   4  * Licensed under the Flora License, Version 1.1 (the License);
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://floralicense.org/license/
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an AS IS BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17
  18 #ifndef _WAKEUP_MANAGER_H_
  19 #define _WAKEUP_MANAGER_H_
  20
  21 #include "service_config.h"
  22 #include "wakeup_manager_wrapper.h"
  23 #include "wakeup_settings.h"
  24 #include "wakeup_engine_manager.h"
  25 #include "wakeup_audio_manager.h"
  26 #include "wakeup_policy_default.h"
  27 #include "assistant_config_manager.h"
  28
  29 #include <memory>
  30 #include <map>
  31 #include <mutex>
  32
  33 namespace multiassistant
  34 {
  35 namespace wakeup
  36 {
  37
  38 using namespace std;
  39
  40 enum class STREAMING_MODE {
  41         NONE,
  42         UTTERANCE,
  43         PREVIOUS_UTTERANCE,
  44         FOLLOW_UP,
  45 };
  46
  47 class IWakeupEventObserver {
  48 public:
  49         virtual ~IWakeupEventObserver() = default;
  50         virtual void on_wakeup(mas_wakeup_event_info wakeup_info) = 0;
  51         virtual void on_streaming_audio_data(
  52                 mas_speech_streaming_event_e event, void* buffer, unsigned int len) = 0;
  53         virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0;
  54         virtual void on_wakeup_engine_command(
  55                 mas_wakeup_engine_command_target_e target, const char* name, const char* command) = 0;
  56         virtual void on_wakeup_service_state_changed(ma_service_state_e state) = 0;
  57         virtual void on_voice_key_status_changed(ma_voice_key_status_e status) = 0;
  58 };
  59
  60 class ISettingValueObserver {
  61 public:
  62         virtual ~ISettingValueObserver() = default;
  63         virtual void on_value_changed() = 0;
  64         virtual void on_loaded_wakeup_engine_changed() = 0;
  65 };
  66
  67 /* If a wakeup event is raised by pressing a voice key,
  68         the wakeup_engine attribute of wakeup_event_info will have the following value */
  69 #define WAKEUP_ENGINE_VOICE_KEY "voice_key"
  70
  71 class CWakeupManager
  72 {
  73 public:
  74         CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer);
  75         virtual ~CWakeupManager();
  76
  77         CWakeupManager(const CWakeupManager&) = delete;
  78         CWakeupManager& operator=(const CWakeupManager&) = delete;
  79
  80         bool initialize();
  81         bool deinitialize();
  82
  83         void subscribe_wakeup_observer(IWakeupEventObserver* observer);
  84         void unsubscribe_wakeup_observer(IWakeupEventObserver* observer);
  85
  86         void subscribe_setting_observer(ISettingValueObserver* observer);
  87         void unsubscribe_setting_observer(ISettingValueObserver* observer);
  88
  89         bool activate();
  90         bool deactivate();
  91
  92         bool add_assistant_language(string appid, string language);
  93         bool add_assistant_wakeup_word(string appid, string wakeup_word, string language);
  94         bool remove_assistant_wakeup_word(string appid, string wakeup_word, string language);
  95         bool add_assistant_wakeup_engine(string appid, string engine);
  96         bool set_assistant_language(string appid, string language);
  97
  98         bool set_assistant_enabled(string appid, bool enabled);
  99         bool get_assistant_enabled(string appid);
 100         bool set_default_assistant(string appid);
 101         bool process_default_assistant_changed(string appid, bool expected);
 102         string get_default_assistant();
 103
 104         bool update_voice_feedback_state(string appid, bool state);
 105         bool set_assistant_specific_command(string appid, string command);
 106         bool set_background_volume(string appid, double ratio);
 107         bool update_recognition_result(string appid, int result);
 108         bool process_plugin_event(mas_plugin_event_e event, void* data, int len);
 109         bool get_audio_format(int* rate, int* channel, int* audio_type);
 110         bool get_audio_source_type(char** type);
 111         bool set_language(string language);
 112         bool get_recording_by_voice_key();
 113         void set_recording_by_voice_key(bool recording);
 114         bool set_voice_key_tap_duration(float duration);
 115         bool unset_voice_key_tap_duration();
 116         bool set_voice_key_support_mode(VOICE_KEY_SUPPORT_MODE mode);
 117         bool set_wake_word_audio_require_flag(bool require);
 118
 119         STREAMING_MODE get_streaming_mode();
 120         bool set_streaming_mode(STREAMING_MODE mode);
 121
 122         bool start_streaming_utterance_data();
 123         bool stop_streaming_utterance_data();
 124         bool start_streaming_follow_up_data();
 125         bool stop_streaming_follow_up_data();
 126         bool start_streaming_previous_utterance_data();
 127         bool stop_streaming_previous_utterance_data();
 128
 129         CWakeupPolicy* get_wakeup_policy();
 130         CWakeupEngineManager* get_engine_manager();
 131         CAudioManager* get_audio_manager();
 132         CWakeupSettings* get_wakeup_settings();
 133
 134         vector<IWakeupEventObserver*> get_wakeup_observers();
 135         void set_last_wakeup_event_info(mas_wakeup_event_info wakeup_info);
 136
 137         vector<ISettingValueObserver*> get_setting_observers();
 138
 139         bool change_manager_state(wakeup_manager_state_e state);
 140         wakeup_manager_state_e get_manager_state();
 141
 142         void feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len);
 143         void set_dependency_module_command(string engine_name, string command);
 144
 145         void start_streaming_duration_timer();
 146         void stop_streaming_duration_timer();
 147
 148         void set_streaming_duration_timer(Ecore_Timer* timer);
 149         Ecore_Timer* get_streaming_duration_timer();
 150
 151         void start_periodic_monitor_timer();
 152         void stop_periodic_monitor_timer();
 153
 154 private:
 155         class CEngineEventObserver : public IEngineEventObserver
 156         {
 157         public:
 158                 bool on_wakeup_event(string engine_name, mas_wakeup_event_info wakeup_info) override;
 159                 bool on_speech_status(string engine_name, mas_speech_status_e status) override;
 160                 bool on_error(string engine_name, int error_code, string error_message) override;
 161                 bool on_audio_data_require_status(string engine_name, bool require) override;
 162
 163                 bool on_streaming_audio_data(
 164                         mas_speech_streaming_event_e event, void* buffer, unsigned int len) override;
 165                 bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override;
 166                 bool on_wakeup_engine_command(
 167                         mas_wakeup_engine_command_target_e target, string engine, string name, string command) override;
 168
 169                 void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; }
 170         private:
 171                 CWakeupManager *mWakeupManager{nullptr};
 172         };
 173
 174         class CPolicyEventObserver : public IPolicyEventObserver
 175         {
 176         public:
 177                 void on_wakeup(mas_wakeup_event_info wakeup_info) override;
 178
 179                 void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; }
 180         private:
 181                 CWakeupManager *mWakeupManager{nullptr};
 182         };
 183
 184         class CAudioEventObserver : public IAudioEventObserver
 185         {
 186         public:
 187                 bool on_recording_audio_data(long time, void* data, int len) override;
 188                 bool on_streaming_audio_data(
 189                         mas_speech_streaming_event_e event, void* buffer, unsigned int len) override;
 190
 191                 void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; }
 192                 void set_wakeup_engine_manager(CWakeupEngineManager *manager) { mEngineManager = manager; }
 193         private:
 194                 CWakeupManager *mWakeupManager{nullptr};
 195                 CWakeupEngineManager *mEngineManager{nullptr};
 196         };
 197
 198         class CSettingsEventObserver : public ISettingsEventObserver
 199         {
 200         public:
 201                 bool on_voice_input_language_changed(const char* language) override;
 202                 bool on_assistant_enabled_info_changed(const char* appid, bool enabled) override;
 203                 bool on_default_assistant_appid_changed(const char* appid) override;
 204                 bool on_multiple_mode_changed() override;
 205
 206                 void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; }
 207         private:
 208                 CWakeupManager *mWakeupManager{nullptr};
 209         };
 210
 211         void initialize_wakeup_policy();
 212
 213         typedef struct {
 214                 string appid;
 215                 vector<string> languageList;
 216         } AssistantLanguageInfo;
 217         vector<AssistantLanguageInfo> mAssistantLanguageInfo;
 218         map<string, bool> mAssistantSupportsCurrentLanguage;
 219         map<string, bool> mAssistantEnabled;
 220
 221         vector<IWakeupEventObserver*> mWakeupObservers;
 222         vector<ISettingValueObserver*> mSettingObservers;
 223
 224         unique_ptr<CWakeupPolicy> mWakeupPolicy;
 225
 226         CAudioManager mAudioManager;
 227         CWakeupEngineManager mWakeupEngineManager;
 228         CWakeupSettings mWakeupSettings;
 229         CAssistantConfigManager mAssistantConfigManager;
 230
 231         CAudioEventObserver mAudioEventObserver;
 232         CEngineEventObserver mEngineEventObserver;
 233         CPolicyEventObserver mPolicyEventObserver;
 234         CSettingsEventObserver mSettingsEventObserver;
 235
 236         thread mEngineDataThread;
 237         atomic_bool mStopEngineDataThread{false};
 238
 239         bool mRecordingByVoiceKey{false};
 240
 241         string mCurrentLanguage;
 242         string mCurrentDefaultAssistant;
 243         /* Assume Push-to-talk is the default mode */
 244         VOICE_KEY_SUPPORT_MODE mCurrentVoiceKeySupportMode{VOICE_KEY_SUPPORT_MODE_PUSH_TO_TALK};
 245
 246         STREAMING_MODE mStreamingMode{STREAMING_MODE::NONE};
 247         Ecore_Timer* mStreamingDurationTimer{nullptr};
 248         Ecore_Timer* mPeriodicMonitorTimer{nullptr};
 249
 250         wakeup_manager_state_e mWakeupManagerState{WAKEUP_MANAGER_STATE_INACTIVE};
 251
 252         mas_wakeup_event_info mLastWakeupEventInfo;
 253
 254         mutex mMutex;
 255 };
 256
 257 } // wakeup
 258 } // multiassistant
 259
 260 #endif /* _WAKEUP_MANAGER_H_ */