1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/speech/chrome_speech_recognition_manager_delegate.h"
10 #include "base/bind.h"
11 #include "base/prefs/pref_service.h"
12 #include "base/strings/utf_string_conversions.h"
13 #include "base/synchronization/lock.h"
14 #include "base/threading/thread_restrictions.h"
15 #include "chrome/browser/browser_process.h"
16 #include "chrome/browser/extensions/extension_service.h"
17 #include "chrome/browser/profiles/profile_manager.h"
18 #include "chrome/browser/tab_contents/tab_util.h"
19 #include "chrome/common/pref_names.h"
20 #include "chrome/common/url_constants.h"
21 #include "content/public/browser/browser_thread.h"
22 #include "content/public/browser/notification_registrar.h"
23 #include "content/public/browser/notification_source.h"
24 #include "content/public/browser/notification_types.h"
25 #include "content/public/browser/render_process_host.h"
26 #include "content/public/browser/render_view_host.h"
27 #include "content/public/browser/resource_context.h"
28 #include "content/public/browser/speech_recognition_manager.h"
29 #include "content/public/browser/speech_recognition_session_config.h"
30 #include "content/public/browser/speech_recognition_session_context.h"
31 #include "content/public/browser/web_contents.h"
32 #include "content/public/common/speech_recognition_error.h"
33 #include "content/public/common/speech_recognition_result.h"
34 #include "extensions/browser/view_type_utils.h"
35 #include "net/url_request/url_request_context_getter.h"
38 #include "chrome/installer/util/wmi.h"
41 using content::BrowserThread;
42 using content::SpeechRecognitionManager;
43 using content::WebContents;
49 void TabClosedCallbackOnIOThread(int render_process_id, int render_view_id) {
50 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
52 SpeechRecognitionManager* manager = SpeechRecognitionManager::GetInstance();
53 // |manager| becomes NULL if a browser shutdown happens between the post of
54 // this task (from the UI thread) and this call (on the IO thread). In this
55 // case we just return.
59 manager->AbortAllSessionsForRenderView(render_process_id, render_view_id);
65 // Asynchronously fetches the PC and audio hardware/driver info if
66 // the user has opted into UMA. This information is sent with speech input
67 // requests to the server for identifying and improving quality issues with
68 // specific device configurations.
69 class ChromeSpeechRecognitionManagerDelegate::OptionalRequestInfo
70 : public base::RefCountedThreadSafe<OptionalRequestInfo> {
72 OptionalRequestInfo() : can_report_metrics_(false) {
76 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
77 // UMA opt-in can be checked only from the UI thread, so switch to that.
78 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
79 base::Bind(&OptionalRequestInfo::CheckUMAAndGetHardwareInfo, this));
82 void CheckUMAAndGetHardwareInfo() {
83 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
84 // prefs::kMetricsReportingEnabled is not registered for OS_CHROMEOS.
85 #if !defined(OS_CHROMEOS)
86 if (g_browser_process->local_state()->GetBoolean(
87 prefs::kMetricsReportingEnabled)) {
88 // Access potentially slow OS calls from the FILE thread.
89 BrowserThread::PostTask(BrowserThread::FILE, FROM_HERE,
90 base::Bind(&OptionalRequestInfo::GetHardwareInfo, this));
95 void GetHardwareInfo() {
96 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
97 base::AutoLock lock(lock_);
98 can_report_metrics_ = true;
99 base::string16 device_model =
100 SpeechRecognitionManager::GetInstance()->GetAudioInputDeviceModel();
102 value_ = base::UTF16ToUTF8(
103 installer::WMIComputerSystem::GetModel() + L"|" + device_model);
104 #else // defined(OS_WIN)
105 value_ = base::UTF16ToUTF8(device_model);
106 #endif // defined(OS_WIN)
109 std::string value() {
110 base::AutoLock lock(lock_);
114 bool can_report_metrics() {
115 base::AutoLock lock(lock_);
116 return can_report_metrics_;
120 friend class base::RefCountedThreadSafe<OptionalRequestInfo>;
122 ~OptionalRequestInfo() {}
126 bool can_report_metrics_;
128 DISALLOW_COPY_AND_ASSIGN(OptionalRequestInfo);
131 // Simple utility to get notified when a WebContent (a tab or an extension's
132 // background page) is closed or crashes. The callback will always be called on
134 // There is no restriction on the constructor, however this class must be
135 // destroyed on the UI thread, due to the NotificationRegistrar dependency.
136 class ChromeSpeechRecognitionManagerDelegate::TabWatcher
137 : public base::RefCountedThreadSafe<TabWatcher>,
138 public content::NotificationObserver {
140 typedef base::Callback<void(int render_process_id, int render_view_id)>
143 explicit TabWatcher(TabClosedCallback tab_closed_callback)
144 : tab_closed_callback_(tab_closed_callback) {
147 // Starts monitoring the WebContents corresponding to the given
148 // |render_process_id|, |render_view_id| pair, invoking |tab_closed_callback_|
149 // if closed/unloaded.
150 void Watch(int render_process_id, int render_view_id) {
151 if (!BrowserThread::CurrentlyOn(BrowserThread::UI)) {
152 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
153 &TabWatcher::Watch, this, render_process_id, render_view_id));
157 WebContents* web_contents = tab_util::GetWebContentsByID(render_process_id,
159 // Sessions initiated by speech input extension APIs will end up in a NULL
160 // WebContent here, but they are properly managed by the
161 // chrome::SpeechInputExtensionManager. However, sessions initiated within a
162 // extension using the (new) speech JS APIs, will be properly handled here.
163 // TODO(primiano) turn this line into a DCHECK once speech input extension
164 // API is deprecated.
168 // Avoid multiple registrations on |registrar_| for the same |web_contents|.
169 if (FindWebContents(web_contents) != registered_web_contents_.end()) {
172 registered_web_contents_.push_back(
173 WebContentsInfo(web_contents, render_process_id, render_view_id));
175 // Lazy initialize the registrar.
176 if (!registrar_.get())
177 registrar_.reset(new content::NotificationRegistrar());
179 registrar_->Add(this,
180 content::NOTIFICATION_RENDER_VIEW_HOST_CHANGED,
181 content::Source<WebContents>(web_contents));
182 registrar_->Add(this,
183 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED,
184 content::Source<WebContents>(web_contents));
187 // content::NotificationObserver implementation.
188 virtual void Observe(int type,
189 const content::NotificationSource& source,
190 const content::NotificationDetails& details) OVERRIDE {
191 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
192 DCHECK(type == content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED ||
193 type == content::NOTIFICATION_RENDER_VIEW_HOST_CHANGED);
195 WebContents* web_contents = content::Source<WebContents>(source).ptr();
196 std::vector<WebContentsInfo>::iterator iter = FindWebContents(web_contents);
197 DCHECK(iter != registered_web_contents_.end());
198 int render_process_id = iter->render_process_id;
199 int render_view_id = iter->render_view_id;
200 registered_web_contents_.erase(iter);
202 registrar_->Remove(this,
203 content::NOTIFICATION_RENDER_VIEW_HOST_CHANGED,
204 content::Source<WebContents>(web_contents));
205 registrar_->Remove(this,
206 content::NOTIFICATION_WEB_CONTENTS_DISCONNECTED,
207 content::Source<WebContents>(web_contents));
209 tab_closed_callback_.Run(render_process_id, render_view_id);
213 struct WebContentsInfo {
214 WebContentsInfo(content::WebContents* web_contents,
215 int render_process_id,
217 : web_contents(web_contents),
218 render_process_id(render_process_id),
219 render_view_id(render_view_id) {}
221 ~WebContentsInfo() {}
223 content::WebContents* web_contents;
224 int render_process_id;
228 friend class base::RefCountedThreadSafe<TabWatcher>;
230 virtual ~TabWatcher() {
231 // Must be destroyed on the UI thread due to |registrar_| non thread-safety.
232 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
235 // Helper function to find the iterator in |registered_web_contents_| which
236 // contains |web_contents|.
237 std::vector<WebContentsInfo>::iterator FindWebContents(
238 content::WebContents* web_contents) {
239 for (std::vector<WebContentsInfo>::iterator i(
240 registered_web_contents_.begin());
241 i != registered_web_contents_.end(); ++i) {
242 if (i->web_contents == web_contents)
246 return registered_web_contents_.end();
249 // Lazy-initialized and used on the UI thread to handle web contents
250 // notifications (tab closing).
251 scoped_ptr<content::NotificationRegistrar> registrar_;
253 // Keeps track of which WebContent(s) have been registered, in order to avoid
254 // double registrations on |registrar_| and to pass the correct render
255 // process id and render view id to |tab_closed_callback_| after the process
257 std::vector<WebContentsInfo> registered_web_contents_;
259 // Callback used to notify, on the thread specified by |callback_thread_| the
260 // closure of a registered tab.
261 TabClosedCallback tab_closed_callback_;
263 DISALLOW_COPY_AND_ASSIGN(TabWatcher);
266 ChromeSpeechRecognitionManagerDelegate
267 ::ChromeSpeechRecognitionManagerDelegate() {
270 ChromeSpeechRecognitionManagerDelegate
271 ::~ChromeSpeechRecognitionManagerDelegate() {
274 void ChromeSpeechRecognitionManagerDelegate::TabClosedCallback(
275 int render_process_id, int render_view_id) {
276 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
278 // Tell the S.R. Manager (which lives on the IO thread) to abort all the
279 // sessions for the given renderer view.
280 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
281 &TabClosedCallbackOnIOThread, render_process_id, render_view_id));
284 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionStart(
286 const content::SpeechRecognitionSessionContext& context =
287 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id);
289 // Register callback to auto abort session on tab closure.
290 // |tab_watcher_| is lazyly istantiated on the first call.
291 if (!tab_watcher_.get()) {
292 tab_watcher_ = new TabWatcher(
293 base::Bind(&ChromeSpeechRecognitionManagerDelegate::TabClosedCallback,
294 base::Unretained(this)));
296 tab_watcher_->Watch(context.render_process_id, context.render_view_id);
299 void ChromeSpeechRecognitionManagerDelegate::OnAudioStart(int session_id) {
302 void ChromeSpeechRecognitionManagerDelegate::OnEnvironmentEstimationComplete(
306 void ChromeSpeechRecognitionManagerDelegate::OnSoundStart(int session_id) {
309 void ChromeSpeechRecognitionManagerDelegate::OnSoundEnd(int session_id) {
312 void ChromeSpeechRecognitionManagerDelegate::OnAudioEnd(int session_id) {
315 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionResults(
316 int session_id, const content::SpeechRecognitionResults& result) {
319 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionError(
320 int session_id, const content::SpeechRecognitionError& error) {
323 void ChromeSpeechRecognitionManagerDelegate::OnAudioLevelsChange(
324 int session_id, float volume, float noise_volume) {
327 void ChromeSpeechRecognitionManagerDelegate::OnRecognitionEnd(int session_id) {
330 void ChromeSpeechRecognitionManagerDelegate::GetDiagnosticInformation(
331 bool* can_report_metrics,
332 std::string* hardware_info) {
333 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
334 if (!optional_request_info_.get()) {
335 optional_request_info_ = new OptionalRequestInfo();
336 // Since hardware info is optional with speech input requests, we start an
337 // asynchronous fetch here and move on with recording audio. This first
338 // speech input request would send an empty string for hardware info and
339 // subsequent requests may have the hardware info available if the fetch
340 // completed before them. This way we don't end up stalling the user with
341 // a long wait and disk seeks when they click on a UI element and start
343 optional_request_info_->Refresh();
345 *can_report_metrics = optional_request_info_->can_report_metrics();
346 *hardware_info = optional_request_info_->value();
349 void ChromeSpeechRecognitionManagerDelegate::CheckRecognitionIsAllowed(
351 base::Callback<void(bool ask_user, bool is_allowed)> callback) {
352 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
354 const content::SpeechRecognitionSessionContext& context =
355 SpeechRecognitionManager::GetInstance()->GetSessionContext(session_id);
357 // Make sure that initiators (extensions/web pages) properly set the
358 // |render_process_id| field, which is needed later to retrieve the profile.
359 DCHECK_NE(context.render_process_id, 0);
361 int render_process_id = context.render_process_id;
362 int render_view_id = context.render_view_id;
363 if (context.embedder_render_process_id) {
364 // If this is a request originated from a guest, we need to re-route the
365 // permission check through the embedder (app).
366 render_process_id = context.embedder_render_process_id;
367 render_view_id = context.embedder_render_view_id;
370 // Check that the render view type is appropriate, and whether or not we
371 // need to request permission from the user.
372 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
373 base::Bind(&CheckRenderViewType,
379 content::SpeechRecognitionEventListener*
380 ChromeSpeechRecognitionManagerDelegate::GetEventListener() {
384 bool ChromeSpeechRecognitionManagerDelegate::FilterProfanities(
385 int render_process_id) {
386 content::RenderProcessHost* rph =
387 content::RenderProcessHost::FromID(render_process_id);
388 if (!rph) // Guard against race conditions on RPH lifetime.
391 return Profile::FromBrowserContext(rph->GetBrowserContext())->GetPrefs()->
392 GetBoolean(prefs::kSpeechRecognitionFilterProfanities);
396 void ChromeSpeechRecognitionManagerDelegate::CheckRenderViewType(
397 base::Callback<void(bool ask_user, bool is_allowed)> callback,
398 int render_process_id,
399 int render_view_id) {
400 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
401 const content::RenderViewHost* render_view_host =
402 content::RenderViewHost::FromID(render_process_id, render_view_id);
404 bool allowed = false;
405 bool check_permission = false;
407 if (!render_view_host) {
408 // This happens for extensions. Manifest should be checked for permission.
410 check_permission = false;
411 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
412 base::Bind(callback, check_permission, allowed));
416 WebContents* web_contents = WebContents::FromRenderViewHost(render_view_host);
418 // chrome://app-list/ uses speech recognition.
419 if (web_contents->GetCommittedWebUI() &&
420 web_contents->GetLastCommittedURL().spec() ==
421 chrome::kChromeUIAppListStartPageURL) {
423 check_permission = false;
426 extensions::ViewType view_type = extensions::GetViewType(web_contents);
428 if (view_type == extensions::VIEW_TYPE_TAB_CONTENTS ||
429 view_type == extensions::VIEW_TYPE_APP_WINDOW ||
430 view_type == extensions::VIEW_TYPE_VIRTUAL_KEYBOARD ||
431 view_type == extensions::VIEW_TYPE_EXTENSION_BACKGROUND_PAGE) {
432 // If it is a tab, we can check for permission. For apps, this means
433 // manifest would be checked for permission.
435 check_permission = true;
438 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
439 base::Bind(callback, check_permission, allowed));
442 } // namespace speech