1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
6 * @fileoverview Accesses Chrome's accessibility extension API and gives
7 * spoken feedback for events that happen in the "Chrome of Chrome".
10 goog.provide('cvox.AccessibilityApiHandler');
12 goog.require('cvox.AbstractEarcons');
13 goog.require('cvox.AbstractTts');
14 goog.require('cvox.BrailleInterface');
15 goog.require('cvox.BrailleUtil');
16 goog.require('cvox.ChromeVoxEditableTextBase');
17 goog.require('cvox.NavBraille');
21 * The chrome.experimental.accessibility API is moving to
22 * chrome.accessibilityPrivate, so provide an alias during the transition.
24 * TODO(dmazzoni): Remove after the stable version of Chrome no longer
25 * has the experimental accessibility API.
27 chrome.experimental = chrome.experimental || {};
29 * Fall back on the experimental API if the new name is not available.
31 chrome.accessibilityPrivate = chrome.accessibilityPrivate ||
32 chrome.experimental.accessibility;
36 * Class that adds listeners and handles events from the accessibility API.
38 * @implements {cvox.TtsCapturingEventListener}
39 * @param {cvox.TtsInterface} tts The TTS to use for speaking.
40 * @param {cvox.BrailleInterface} braille The braille interface to use for
42 * @param {Object} earcons The earcons object to use for playing
45 cvox.AccessibilityApiHandler = function(tts, braille, earcons) {
47 this.braille = braille;
48 this.earcons = earcons;
50 * Tracks the previous description received.
54 this.prevDescription_ = {};
56 * Array of strings to speak the next time TTS is idle.
57 * @type {!Array.<string>}
60 this.idleSpeechQueue_ = [];
63 chrome.accessibilityPrivate.setAccessibilityEnabled(true);
64 chrome.accessibilityPrivate.setNativeAccessibilityEnabled(
65 !cvox.ChromeVox.isActive);
66 this.addEventListeners_();
67 if (cvox.ChromeVox.isActive) {
68 this.queueAlertsForActiveTab();
71 console.log('Error trying to access accessibility extension api.');
76 * The interface used to manage speech.
77 * @type {cvox.TtsInterface}
79 cvox.AccessibilityApiHandler.prototype.tts = null;
82 * The interface used to manage braille.
83 * @type {cvox.BrailleInterface}
85 cvox.AccessibilityApiHandler.prototype.braille = null;
88 * The object used to manage arcons.
91 cvox.AccessibilityApiHandler.prototype.earcons = null;
94 * The object that can describe changes and cursor movement in a generic
95 * editable text field.
98 cvox.AccessibilityApiHandler.prototype.editableTextHandler = null;
101 * The name of the editable text field associated with
102 * |editableTextHandler|, so we can tell when focus moves.
105 cvox.AccessibilityApiHandler.prototype.editableTextName = '';
108 * The queue mode for the next focus event.
111 cvox.AccessibilityApiHandler.prototype.nextQueueMode = 0;
114 * The timeout id for the pending text changed event - the return
115 * value from window.setTimeout. We need to delay text events slightly
116 * and return only the last one because sometimes we get a rapid
117 * succession of related events that should all be considered one
118 * bulk change - in particular, autocomplete in the location bar comes
119 * as multiple events in a row.
122 cvox.AccessibilityApiHandler.prototype.textChangeTimeout = null;
125 * Most controls have a "context" - the name of the window, dialog, toolbar,
126 * or menu they're contained in. We announce a context once, when you
127 * first enter it - and we don't announce it again when you move to something
128 * else within the same context. This variable keeps track of the most
132 cvox.AccessibilityApiHandler.prototype.lastContext = null;
135 * Delay in ms between when a text event is received and when it's spoken.
139 cvox.AccessibilityApiHandler.prototype.TEXT_CHANGE_DELAY = 10;
142 * ID returned from setTimeout to queue up speech on idle.
146 cvox.AccessibilityApiHandler.prototype.idleSpeechTimeout_ = null;
149 * Milliseconds of silence to wait before considering speech to be idle.
152 cvox.AccessibilityApiHandler.prototype.IDLE_SPEECH_DELAY_MS = 500;
155 * Called to let us know that the last speech came from web, and not from
156 * native UI. Clear the context and any state associated with the last
159 cvox.AccessibilityApiHandler.prototype.setWebContext = function() {
160 // This will never be spoken - it's just supposed to be a string that
161 // won't match the context of the next control that gets focused.
162 this.lastContext = '--internal-web--';
163 this.editableTextHandler = null;
164 this.editableTextName = '';
166 if (chrome.accessibilityPrivate.setFocusRing &&
167 cvox.ChromeVox.isChromeOS) {
168 // Clear the focus ring.
169 chrome.accessibilityPrivate.setFocusRing([]);
174 * Adds event listeners.
177 cvox.AccessibilityApiHandler.prototype.addEventListeners_ = function() {
178 /** Alias getMsg as msg. */
179 var msg = goog.bind(cvox.ChromeVox.msgs.getMsg, cvox.ChromeVox.msgs);
181 var accessibility = chrome.accessibilityPrivate;
183 chrome.tabs.onActivated.addListener(goog.bind(function(activeInfo) {
184 if (!cvox.ChromeVox.isActive) {
187 chrome.tabs.get(activeInfo.tabId, goog.bind(function(tab) {
188 if (tab.status == 'loading') {
191 this.queueAlertsForActiveTab();
195 chrome.accessibilityPrivate.onWindowOpened.addListener(
196 goog.bind(function(win) {
197 if (!cvox.ChromeVox.isActive) {
200 this.tts.speak(win.name,
201 cvox.AbstractTts.QUEUE_MODE_FLUSH,
202 cvox.AbstractTts.PERSONALITY_ANNOUNCEMENT);
203 this.braille.write(cvox.NavBraille.fromText(win.name));
204 // Queue the next utterance because a window opening is always followed
206 this.nextQueueMode = 1;
207 this.earcons.playEarcon(cvox.AbstractEarcons.OBJECT_OPEN);
208 this.queueAlertsForActiveTab();
211 chrome.accessibilityPrivate.onWindowClosed.addListener(
212 goog.bind(function(win) {
213 if (!cvox.ChromeVox.isActive) {
216 // Don't speak, just play the earcon.
217 this.earcons.playEarcon(cvox.AbstractEarcons.OBJECT_CLOSE);
220 chrome.accessibilityPrivate.onMenuOpened.addListener(
221 goog.bind(function(menu) {
222 if (!cvox.ChromeVox.isActive) {
225 this.tts.speak(msg('chrome_menu_opened', [menu.name]),
226 cvox.AbstractTts.QUEUE_MODE_FLUSH,
227 cvox.AbstractTts.PERSONALITY_ANNOUNCEMENT);
229 cvox.NavBraille.fromText(msg('chrome_menu_opened', [menu.name])));
230 this.earcons.playEarcon(cvox.AbstractEarcons.OBJECT_OPEN);
233 chrome.accessibilityPrivate.onMenuClosed.addListener(
234 goog.bind(function(menu) {
235 if (!cvox.ChromeVox.isActive) {
238 // Don't speak, just play the earcon.
239 this.earcons.playEarcon(cvox.AbstractEarcons.OBJECT_CLOSE);
242 // systemPrivate API is only available when this extension is loaded as a
243 // component extension embedded in Chrome.
244 chrome.permissions.contains(
245 { permissions: ['systemPrivate'] },
246 goog.bind(function(result) {
251 // TODO(plundblad): Remove when the native sound is turned on by default.
252 // See crbug.com:225886.
253 var addOnVolumeChangedListener = goog.bind(function() {
254 chrome.systemPrivate.onVolumeChanged.addListener(goog.bind(
256 if (!cvox.ChromeVox.isActive) {
259 // Don't speak, just play the earcon.
260 this.earcons.playEarcon(cvox.AbstractEarcons.TASK_SUCCESS);
263 if (chrome.commandLinePrivate) {
264 chrome.commandLinePrivate.hasSwitch('disable-volume-adjust-sound',
265 goog.bind(function(result) {
267 addOnVolumeChangedListener();
271 addOnVolumeChangedListener();
274 chrome.systemPrivate.onBrightnessChanged.addListener(
277 * @param {{brightness: number, userInitiated: boolean}} brightness
279 function(brightness) {
280 if (brightness.userInitiated) {
281 this.earcons.playEarcon(cvox.AbstractEarcons.TASK_SUCCESS);
283 msg('chrome_brightness_changed', [brightness.brightness]),
284 cvox.AbstractTts.QUEUE_MODE_FLUSH,
285 cvox.AbstractTts.PERSONALITY_ANNOUNCEMENT);
286 this.braille.write(cvox.NavBraille.fromText(
287 msg('chrome_brightness_changed', [brightness.brightness])));
291 chrome.systemPrivate.onScreenUnlocked.addListener(goog.bind(function() {
292 chrome.systemPrivate.getUpdateStatus(goog.bind(function(status) {
293 if (!cvox.ChromeVox.isActive) {
296 // Speak about system update when it's ready, otherwise speak nothing.
297 if (status.state == 'NeedRestart') {
298 this.tts.speak(msg('chrome_system_need_restart'),
299 cvox.AbstractTts.QUEUE_MODE_FLUSH,
300 cvox.AbstractTts.PERSONALITY_ANNOUNCEMENT);
302 cvox.NavBraille.fromText(msg('chrome_system_need_restart')));
307 chrome.systemPrivate.onWokeUp.addListener(goog.bind(function() {
308 if (!cvox.ChromeVox.isActive) {
311 // Don't speak, just play the earcon.
312 this.earcons.playEarcon(cvox.AbstractEarcons.OBJECT_OPEN);
316 chrome.accessibilityPrivate.onControlFocused.addListener(
317 goog.bind(this.onControlFocused, this));
319 chrome.accessibilityPrivate.onControlAction.addListener(
320 goog.bind(function(ctl) {
321 if (!cvox.ChromeVox.isActive) {
325 var description = this.describe(ctl, true);
326 this.tts.speak(description.utterance,
327 cvox.AbstractTts.QUEUE_MODE_FLUSH,
328 description.ttsProps);
329 description.braille.write();
330 if (description.earcon) {
331 this.earcons.playEarcon(description.earcon);
336 chrome.accessibilityPrivate.onControlHover.addListener(
337 goog.bind(function(ctl) {
338 if (!cvox.ChromeVox.isActive) {
342 var hasTouch = 'ontouchstart' in window;
347 var description = this.describe(ctl, false);
348 this.tts.speak(description.utterance,
349 cvox.AbstractTts.QUEUE_MODE_FLUSH,
350 description.ttsProps);
351 description.braille.write();
352 if (description.earcon) {
353 this.earcons.playEarcon(description.earcon);
358 chrome.accessibilityPrivate.onTextChanged.addListener(
359 goog.bind(function(ctl) {
360 if (!cvox.ChromeVox.isActive) {
364 if (!this.editableTextHandler ||
365 this.editableTextName != ctl.name ||
366 this.lastContext != ctl.context) {
367 // Chrome won't send a text change event on a control that isn't
368 // focused. If we get a text change event and it doesn't match the
369 // focused control, treat it as a focus event initially.
370 this.onControlFocused(ctl);
374 // Only send the most recent text changed event - throw away anything
376 if (this.textChangeTimeout) {
377 window.clearTimeout(this.textChangeTimeout);
380 // Handle the text change event after a small delay, so multiple
381 // events in rapid succession are handled as a single change. This is
382 // specifically for the location bar with autocomplete - typing a
383 // character and getting the autocompleted text and getting that
384 // text selected may be three separate events.
385 this.textChangeTimeout = window.setTimeout(
386 goog.bind(function() {
387 var textChangeEvent = new cvox.TextChangeEvent(
389 ctl.details.selectionStart,
390 ctl.details.selectionEnd,
391 true); // triggered by user
392 this.editableTextHandler.changed(
394 this.describe(ctl, false).braille.write();
395 }, this), this.TEXT_CHANGE_DELAY);
398 this.tts.addCapturingEventListener(this);
402 * Handle the feedback when a new control gets focus.
403 * @param {AccessibilityObject} ctl The focused control.
405 cvox.AccessibilityApiHandler.prototype.onControlFocused = function(ctl) {
406 if (!cvox.ChromeVox.isActive) {
411 chrome.accessibilityPrivate.setFocusRing &&
412 cvox.ChromeVox.isChromeOS) {
413 chrome.accessibilityPrivate.setFocusRing([ctl.bounds]);
416 // Call this first because it may clear this.editableTextHandler.
417 var description = this.describe(ctl, false);
419 if (ctl.type == 'textbox') {
420 var start = ctl.details.selectionStart;
421 var end = ctl.details.selectionEnd;
423 start = ctl.details.selectionEnd;
424 end = ctl.details.selectionStart;
426 this.editableTextName = ctl.name;
427 this.editableTextHandler =
428 new cvox.ChromeVoxEditableTextBase(
432 ctl.details.isPassword,
435 this.editableTextHandler = null;
438 this.tts.speak(description.utterance,
440 description.ttsProps);
441 description.braille.write();
442 this.nextQueueMode = 0;
443 if (description.earcon) {
444 this.earcons.playEarcon(description.earcon);
449 * Called when any speech starts.
451 cvox.AccessibilityApiHandler.prototype.onTtsStart = function() {
452 if (this.idleSpeechTimeout_) {
453 window.clearTimeout(this.idleSpeechTimeout_);
458 * Called when any speech ends.
460 cvox.AccessibilityApiHandler.prototype.onTtsEnd = function() {
461 if (this.idleSpeechQueue_.length > 0) {
462 this.idleSpeechTimeout_ = window.setTimeout(
463 goog.bind(this.onTtsIdle, this),
464 this.IDLE_SPEECH_DELAY_MS);
469 * Called when speech has been idle for a certain minimum delay.
470 * Speaks queued messages.
472 cvox.AccessibilityApiHandler.prototype.onTtsIdle = function() {
473 if (this.idleSpeechQueue_.length == 0) {
476 var utterance = this.idleSpeechQueue_.shift();
477 var msg = goog.bind(cvox.ChromeVox.msgs.getMsg, cvox.ChromeVox.msgs);
478 this.tts.speak(utterance,
479 cvox.AbstractTts.QUEUE_MODE_FLUSH,
480 cvox.AbstractTts.PERSONALITY_ANNOUNCEMENT);
484 * Given a control received from the accessibility api, determine an
485 * utterance to speak, text to braille, and an earcon to play to describe it.
486 * @param {Object} control The control that had an action performed on it.
487 * @param {boolean} isSelect True if the action is a select action,
488 * otherwise it's a focus action.
489 * @return {Object} An object containing a string field |utterance|, object
490 * |ttsProps|, |braille|, and earcon |earcon|.
492 cvox.AccessibilityApiHandler.prototype.describe = function(control, isSelect) {
493 /** Alias getMsg as msg. */
494 var msg = goog.bind(cvox.ChromeVox.msgs.getMsg, cvox.ChromeVox.msgs);
498 var ttsProps = cvox.AbstractTts.PERSONALITY_ANNOUNCEMENT;
500 var context = control.context;
501 if (context && context != this.lastContext) {
503 this.lastContext = context;
504 this.editableTextHandler = null;
507 var earcon = undefined;
508 var name = control.name.replace(/[_&]+/g, '').replace('...', '');
509 braille.name = control.name;
510 switch (control.type) {
512 braille.roleMsg = 'input_type_checkbox';
513 if (control.details.isChecked) {
514 earcon = cvox.AbstractEarcons.CHECK_ON;
515 s += msg('describe_checkbox_checked', [name]);
516 braille.state = msg('checkbox_checked_state_brl');
518 earcon = cvox.AbstractEarcons.CHECK_OFF;
519 s += msg('describe_checkbox_unchecked', [name]);
520 braille.state = msg('checkbox_unchecked_state_brl');
525 braille.roleMsg = 'input_type_radio';
526 if (control.details.isChecked) {
527 earcon = cvox.AbstractEarcons.CHECK_ON;
528 s += msg('describe_radio_selected', [name]);
529 braille.state = msg('radio_selected_state_brl');
531 earcon = cvox.AbstractEarcons.CHECK_OFF;
532 s += msg('describe_radio_unselected', [name]);
533 braille.state = msg('radio_unselected_state_brl');
537 s += msg('describe_menu', [name]);
538 braille.roleMsg = 'aria_role_menu';
542 control.details.hasSubmenu ?
543 'describe_menu_item_with_submenu' : 'describe_menu_item', [name]);
544 braille.roleMsg = 'aria_role_menuitem';
545 if (control.details.hasSubmenu) {
546 braille.state = msg('aria_has_submenu_brl');
550 s += msg('describe_window', [name]);
551 // No specialization for braille.
555 earcon = cvox.AbstractEarcons.ALERT_NONMODAL;
556 s += msg('aria_role_alert') + ': ' + name;
557 ttsProps = cvox.AbstractTts.PERSONALITY_SYSTEM_ALERT;
558 braille.roleMsg = 'aria_role_alert';
562 earcon = cvox.AbstractEarcons.EDITABLE_TEXT;
563 var unnamed = name == '' ? 'unnamed_' : '';
565 if (control.details.isPassword) {
567 braille.roleMsg = 'input_type_password';
568 value = control.details.value.replace(/./g, '*');
571 braille.roleMsg = 'input_type_text';
572 value = control.details.value;
574 s += msg('describe_' + unnamed + type, [value, name]);
575 braille.value = cvox.BrailleUtil.createValue(
576 value, control.details.selectionStart, control.details.selectionEnd);
579 earcon = cvox.AbstractEarcons.BUTTON;
580 s += msg('describe_button', [name]);
581 braille.roleMsg = 'tag_button';
588 earcon = cvox.AbstractEarcons.LISTBOX;
589 var unnamed = name == '' ? 'unnamed_' : '';
590 s += msg('describe_' + unnamed + control.type,
591 [control.details.value, name]);
592 braille.roleMsg = 'tag_select';
595 earcon = cvox.AbstractEarcons.LINK;
596 s += msg('describe_link', [name]);
597 braille.roleMsg = 'tag_link';
600 s += msg('describe_tab', [name]);
601 braille.roleMsg = 'aria_role_tab';
604 s += msg('describe_slider', [control.details.stringValue, name]);
605 braille.value = cvox.BrailleUtil.createValue(control.details.stringValue);
606 braille.roleMsg = 'aria_role_slider';
609 if (this.prevDescription_ &&
610 this.prevDescription_.details &&
611 goog.isDef(control.details.itemDepth) &&
612 this.prevDescription_.details.itemDepth !=
613 control.details.itemDepth) {
614 s += msg('describe_depth', [control.details.itemDepth]);
616 s += name + ' ' + msg('aria_role_treeitem');
617 s += control.details.isItemExpanded ?
618 msg('aria_expanded_true') : msg('aria_expanded_false');
620 braille.name = Array(control.details.itemDepth).join(' ') + braille.name;
621 braille.roleMsg = 'aria_role_treeitem';
622 braille.state = control.details.isItemExpanded ?
623 msg('aria_expanded_true_brl') : msg('aria_expanded_false_brl');
627 s += name + ', ' + control.type;
628 braille.role = control.type;
631 if (isSelect && control.type != 'slider') {
632 s += msg('describe_selected');
634 if (control.details && control.details.itemCount >= 0) {
635 s += msg('describe_index',
636 [control.details.itemIndex + 1, control.details.itemCount]);
637 braille.state = braille.state ? braille.state + ' ' : '';
638 braille.state += msg('LIST_POSITION_BRL',
639 [control.details.itemIndex + 1, control.details.itemCount]);
642 var description = {};
643 description.utterance = s;
644 description.ttsProps = ttsProps;
645 var spannable = cvox.BrailleUtil.getTemplated(null, null, braille);
646 var valueSelectionSpan = spannable.getSpanInstanceOf(
647 cvox.BrailleUtil.ValueSelectionSpan);
648 var brailleObj = {text: spannable};
649 if (valueSelectionSpan) {
650 brailleObj.startIndex = spannable.getSpanStart(valueSelectionSpan);
651 brailleObj.endIndex = spannable.getSpanEnd(valueSelectionSpan);
653 description.braille = new cvox.NavBraille(brailleObj);
654 description.earcon = earcon;
655 this.prevDescription_ = control;
660 * Queues alerts for the active tab, if any, which will be spoken
661 * as soon as speech is idle.
663 cvox.AccessibilityApiHandler.prototype.queueAlertsForActiveTab = function() {
664 this.idleSpeechQueue_.length = 0;
665 var msg = goog.bind(cvox.ChromeVox.msgs.getMsg, cvox.ChromeVox.msgs);
667 chrome.tabs.query({'active': true, 'currentWindow': true},
668 goog.bind(function(tabs) {
669 if (tabs.length < 1) {
672 chrome.accessibilityPrivate.getAlertsForTab(
673 tabs[0].id, goog.bind(function(alerts) {
674 if (alerts.length == 0) {
680 if (alerts.length == 1) {
681 utterance += msg('page_has_one_alert_singular');
683 utterance += msg('page_has_alerts_plural',
687 for (var i = 0; i < alerts.length; i++) {
688 utterance += ' ' + alerts[i].message;
691 utterance += ' ' + msg('review_alerts');
693 if (this.idleSpeechQueue_.indexOf(utterance) == -1) {
694 this.idleSpeechQueue_.push(utterance);