1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
9 #include "base/command_line.h"
10 #include "base/file_util.h"
11 #include "base/logging.h"
12 #include "base/path_service.h"
13 #include "base/strings/string16.h"
14 #include "base/strings/string_split.h"
15 #include "base/strings/string_util.h"
16 #include "base/strings/utf_string_conversions.h"
17 #include "content/browser/accessibility/accessibility_tree_formatter.h"
18 #include "content/browser/accessibility/browser_accessibility.h"
19 #include "content/browser/accessibility/browser_accessibility_manager.h"
20 #include "content/browser/renderer_host/render_view_host_impl.h"
21 #include "content/browser/renderer_host/render_widget_host_view_base.h"
22 #include "content/public/browser/web_contents.h"
23 #include "content/public/common/content_paths.h"
24 #include "content/public/common/content_switches.h"
25 #include "content/public/common/url_constants.h"
26 #include "content/public/test/content_browser_test.h"
27 #include "content/public/test/content_browser_test_utils.h"
28 #include "content/shell/browser/shell.h"
29 #include "content/test/accessibility_browser_test_utils.h"
30 #include "testing/gtest/include/gtest/gtest.h"
32 // TODO(aboxhall): Create expectations on Android for these
33 #if defined(OS_ANDROID)
34 #define MAYBE(x) DISABLED_##x
43 const char kCommentToken = '#';
44 const char kMarkSkipFile[] = "#<skip";
45 const char kMarkEndOfFile[] = "<-- End-of-file -->";
46 const char kSignalDiff[] = "*";
50 typedef AccessibilityTreeFormatter::Filter Filter;
52 // This test takes a snapshot of the platform BrowserAccessibility tree and
53 // tests it against an expected baseline.
55 // The flow of the test is as outlined below.
56 // 1. Load an html file from chrome/test/data/accessibility.
57 // 2. Read the expectation.
58 // 3. Browse to the page and serialize the platform specific tree into a human
60 // 4. Perform a comparison between actual and expected and fail if they do not
62 class DumpAccessibilityTreeTest : public ContentBrowserTest {
64 // Utility helper that does a comment aware equality check.
65 // Returns array of lines from expected file which are different.
66 std::vector<int> DiffLines(const std::vector<std::string>& expected_lines,
67 const std::vector<std::string>& actual_lines) {
68 int actual_lines_count = actual_lines.size();
69 int expected_lines_count = expected_lines.size();
70 std::vector<int> diff_lines;
72 while (i < actual_lines_count && j < expected_lines_count) {
73 if (expected_lines[j].size() == 0 ||
74 expected_lines[j][0] == kCommentToken) {
75 // Skip comment lines and blank lines in expected output.
80 if (actual_lines[i] != expected_lines[j])
81 diff_lines.push_back(j);
86 // Actual file has been fully checked.
90 void AddDefaultFilters(std::vector<Filter>* filters) {
91 filters->push_back(Filter(base::ASCIIToUTF16("FOCUSABLE"), Filter::ALLOW));
92 filters->push_back(Filter(base::ASCIIToUTF16("READONLY"), Filter::ALLOW));
93 filters->push_back(Filter(base::ASCIIToUTF16("*=''"), Filter::DENY));
96 // Parse the test html file and parse special directives, usually
97 // beginning with an '@' and inside an HTML comment, that control how the
98 // test is run and how the results are interpreted.
100 // When the accessibility tree is dumped as text, each attribute is
101 // run through filters before being appended to the string. An "allow"
102 // filter specifies attribute strings that should be dumped, and a "deny"
103 // filter specifies strings that should be suppressed. As an example,
104 // @MAC-ALLOW:AXSubrole=* means that the AXSubrole attribute should be
105 // printed, while @MAC-ALLOW:AXSubrole=AXList* means that any subrole
106 // beginning with the text "AXList" should be printed.
108 // The @WAIT-FOR:text directive allows the test to specify that the document
109 // may dynamically change after initial load, and the test is to wait
110 // until the given string (e.g., "text") appears in the resulting dump.
111 // A test can make some changes to the document, then append a magic string
112 // indicating that the test is done, and this framework will wait for that
113 // string to appear before comparing the results.
114 void ParseHtmlForExtraDirectives(const std::string& test_html,
115 std::vector<Filter>* filters,
116 std::string* wait_for) {
117 std::vector<std::string> lines;
118 base::SplitString(test_html, '\n', &lines);
119 for (std::vector<std::string>::const_iterator iter = lines.begin();
122 const std::string& line = *iter;
123 const std::string& allow_empty_str =
124 AccessibilityTreeFormatter::GetAllowEmptyString();
125 const std::string& allow_str =
126 AccessibilityTreeFormatter::GetAllowString();
127 const std::string& deny_str =
128 AccessibilityTreeFormatter::GetDenyString();
129 const std::string& wait_str = "@WAIT-FOR:";
130 if (StartsWithASCII(line, allow_empty_str, true)) {
132 Filter(base::UTF8ToUTF16(line.substr(allow_empty_str.size())),
133 Filter::ALLOW_EMPTY));
134 } else if (StartsWithASCII(line, allow_str, true)) {
135 filters->push_back(Filter(base::UTF8ToUTF16(
136 line.substr(allow_str.size())),
138 } else if (StartsWithASCII(line, deny_str, true)) {
139 filters->push_back(Filter(base::UTF8ToUTF16(
140 line.substr(deny_str.size())),
142 } else if (StartsWithASCII(line, wait_str, true)) {
143 *wait_for = line.substr(wait_str.size());
148 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
149 ContentBrowserTest::SetUpCommandLine(command_line);
150 // Enable <dialog>, which is used in some tests.
151 CommandLine::ForCurrentProcess()->AppendSwitch(
152 switches::kEnableExperimentalWebPlatformFeatures);
155 void RunTest(const base::FilePath::CharType* file_path);
158 void DumpAccessibilityTreeTest::RunTest(
159 const base::FilePath::CharType* file_path) {
160 NavigateToURL(shell(), GURL(url::kAboutBlankURL));
163 base::FilePath dir_test_data;
164 ASSERT_TRUE(PathService::Get(DIR_TEST_DATA, &dir_test_data));
165 base::FilePath test_path(
166 dir_test_data.Append(FILE_PATH_LITERAL("accessibility")));
167 ASSERT_TRUE(base::PathExists(test_path))
168 << test_path.LossyDisplayName();
170 base::FilePath html_file = test_path.Append(base::FilePath(file_path));
171 // Output the test path to help anyone who encounters a failure and needs
172 // to know where to look.
173 printf("Testing: %s\n", html_file.MaybeAsASCII().c_str());
175 std::string html_contents;
176 base::ReadFileToString(html_file, &html_contents);
178 // Read the expected file.
179 std::string expected_contents_raw;
180 base::FilePath expected_file =
181 base::FilePath(html_file.RemoveExtension().value() +
182 AccessibilityTreeFormatter::GetExpectedFileSuffix());
183 base::ReadFileToString(expected_file, &expected_contents_raw);
185 // Tolerate Windows-style line endings (\r\n) in the expected file:
186 // normalize by deleting all \r from the file (if any) to leave only \n.
187 std::string expected_contents;
188 base::RemoveChars(expected_contents_raw, "\r", &expected_contents);
190 if (!expected_contents.compare(0, strlen(kMarkSkipFile), kMarkSkipFile)) {
191 printf("Skipping this test on this platform.\n");
195 // Parse filters and other directives in the test file.
196 std::vector<Filter> filters;
197 std::string wait_for;
198 AddDefaultFilters(&filters);
199 ParseHtmlForExtraDirectives(html_contents, &filters, &wait_for);
202 base::string16 html_contents16;
203 html_contents16 = base::UTF8ToUTF16(html_contents);
204 GURL url = GetTestUrl("accessibility",
205 html_file.BaseName().MaybeAsASCII().c_str());
207 // If there's a @WAIT-FOR directive, set up an accessibility notification
208 // waiter that returns on any event; we'll stop when we get the text we're
209 // waiting for, or time out. Otherwise just wait specifically for
210 // the "load complete" event.
211 scoped_ptr<AccessibilityNotificationWaiter> waiter;
212 if (!wait_for.empty()) {
213 waiter.reset(new AccessibilityNotificationWaiter(
214 shell(), AccessibilityModeComplete, ui::AX_EVENT_NONE));
216 waiter.reset(new AccessibilityNotificationWaiter(
217 shell(), AccessibilityModeComplete, ui::AX_EVENT_LOAD_COMPLETE));
220 // Load the test html.
221 NavigateToURL(shell(), url);
223 // Wait for notifications. If there's a @WAIT-FOR directive, break when
224 // the text we're waiting for appears in the dump, otherwise break after
225 // the first notification, which will be a load complete.
226 RenderWidgetHostViewBase* host_view = static_cast<RenderWidgetHostViewBase*>(
227 shell()->web_contents()->GetRenderWidgetHostView());
228 std::string actual_contents;
230 waiter->WaitForNotification();
231 base::string16 actual_contents_utf16;
232 AccessibilityTreeFormatter formatter(
233 host_view->GetBrowserAccessibilityManager()->GetRoot());
234 formatter.SetFilters(filters);
235 formatter.FormatAccessibilityTree(&actual_contents_utf16);
236 actual_contents = base::UTF16ToUTF8(actual_contents_utf16);
237 } while (!wait_for.empty() &&
238 actual_contents.find(wait_for) == std::string::npos);
240 // Perform a diff (or write the initial baseline).
241 std::vector<std::string> actual_lines, expected_lines;
242 Tokenize(actual_contents, "\n", &actual_lines);
243 Tokenize(expected_contents, "\n", &expected_lines);
244 // Marking the end of the file with a line of text ensures that
245 // file length differences are found.
246 expected_lines.push_back(kMarkEndOfFile);
247 actual_lines.push_back(kMarkEndOfFile);
249 std::vector<int> diff_lines = DiffLines(expected_lines, actual_lines);
250 bool is_different = diff_lines.size() > 0;
251 EXPECT_FALSE(is_different);
253 // Mark the expected lines which did not match actual output with a *.
254 printf("* Line Expected\n");
255 printf("- ---- --------\n");
256 for (int line = 0, diff_index = 0;
257 line < static_cast<int>(expected_lines.size());
259 bool is_diff = false;
260 if (diff_index < static_cast<int>(diff_lines.size()) &&
261 diff_lines[diff_index] == line) {
265 printf("%1s %4d %s\n", is_diff? kSignalDiff : "", line + 1,
266 expected_lines[line].c_str());
268 printf("\nActual\n");
270 printf("%s\n", actual_contents.c_str());
273 if (!base::PathExists(expected_file)) {
274 base::FilePath actual_file =
275 base::FilePath(html_file.RemoveExtension().value() +
276 AccessibilityTreeFormatter::GetActualFileSuffix());
278 EXPECT_TRUE(base::WriteFile(
279 actual_file, actual_contents.c_str(), actual_contents.size()));
281 ADD_FAILURE() << "No expectation found. Create it by doing:\n"
282 << "mv " << actual_file.LossyDisplayName() << " "
283 << expected_file.LossyDisplayName();
287 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityA) {
288 RunTest(FILE_PATH_LITERAL("a.html"));
291 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAddress) {
292 RunTest(FILE_PATH_LITERAL("address.html"));
295 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAName) {
296 RunTest(FILE_PATH_LITERAL("a-name.html"));
299 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityANoText) {
300 RunTest(FILE_PATH_LITERAL("a-no-text.html"));
303 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAOnclick) {
304 RunTest(FILE_PATH_LITERAL("a-onclick.html"));
307 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
308 AccessibilityAriaApplication) {
309 RunTest(FILE_PATH_LITERAL("aria-application.html"));
312 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
313 AccessibilityAriaAutocomplete) {
314 RunTest(FILE_PATH_LITERAL("aria-autocomplete.html"));
317 // crbug.com/98976 will cause new elements to be added to the Blink a11y tree
318 // Re-baseline after the Blink change goes in
319 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
320 DISABLED_AccessibilityAriaCombobox) {
321 RunTest(FILE_PATH_LITERAL("aria-combobox.html"));
324 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
325 MAYBE(AccessibilityAriaFlowto)) {
326 RunTest(FILE_PATH_LITERAL("aria-flowto.html"));
329 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAriaInvalid) {
330 RunTest(FILE_PATH_LITERAL("aria-invalid.html"));
333 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
334 AccessibilityAriaLabelledByHeading) {
335 RunTest(FILE_PATH_LITERAL("aria-labelledby-heading.html"));
338 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAriaLevel) {
339 RunTest(FILE_PATH_LITERAL("aria-level.html"));
342 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAriaList) {
343 RunTest(FILE_PATH_LITERAL("aria-list.html"));
346 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAriaMenu) {
347 RunTest(FILE_PATH_LITERAL("aria-menu.html"));
350 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
351 AccessibilityAriaMenuitemradio) {
352 RunTest(FILE_PATH_LITERAL("aria-menuitemradio.html"));
355 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
356 AccessibilityAriaPressed) {
357 RunTest(FILE_PATH_LITERAL("aria-pressed.html"));
360 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
361 AccessibilityAriaProgressbar) {
362 RunTest(FILE_PATH_LITERAL("aria-progressbar.html"));
365 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
366 AccessibilityAriaToolbar) {
367 RunTest(FILE_PATH_LITERAL("toolbar.html"));
370 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
371 AccessibilityAriaValueMin) {
372 RunTest(FILE_PATH_LITERAL("aria-valuemin.html"));
375 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
376 AccessibilityAriaValueMax) {
377 RunTest(FILE_PATH_LITERAL("aria-valuemax.html"));
380 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityArticle) {
381 RunTest(FILE_PATH_LITERAL("article.html"));
384 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityAWithImg) {
385 RunTest(FILE_PATH_LITERAL("a-with-img.html"));
388 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityBdo) {
389 RunTest(FILE_PATH_LITERAL("bdo.html"));
392 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityBR) {
393 RunTest(FILE_PATH_LITERAL("br.html"));
396 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityButtonNameCalc) {
397 RunTest(FILE_PATH_LITERAL("button-name-calc.html"));
400 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityCanvas) {
401 RunTest(FILE_PATH_LITERAL("canvas.html"));
404 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
405 AccessibilityCheckboxNameCalc) {
406 RunTest(FILE_PATH_LITERAL("checkbox-name-calc.html"));
409 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityDialog) {
410 RunTest(FILE_PATH_LITERAL("dialog.html"));
413 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityDiv) {
414 RunTest(FILE_PATH_LITERAL("div.html"));
417 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityDl) {
418 RunTest(FILE_PATH_LITERAL("dl.html"));
421 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
422 AccessibilityContenteditableDescendants) {
423 RunTest(FILE_PATH_LITERAL("contenteditable-descendants.html"));
426 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityEm) {
427 RunTest(FILE_PATH_LITERAL("em.html"));
430 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityFooter) {
431 RunTest(FILE_PATH_LITERAL("footer.html"));
434 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityForm) {
435 RunTest(FILE_PATH_LITERAL("form.html"));
438 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityHeading) {
439 RunTest(FILE_PATH_LITERAL("heading.html"));
442 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityHR) {
443 RunTest(FILE_PATH_LITERAL("hr.html"));
446 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
447 AccessibilityIframeCoordinates) {
448 RunTest(FILE_PATH_LITERAL("iframe-coordinates.html"));
451 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityInputButton) {
452 RunTest(FILE_PATH_LITERAL("input-button.html"));
455 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
456 AccessibilityInputButtonInMenu) {
457 RunTest(FILE_PATH_LITERAL("input-button-in-menu.html"));
460 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityInputColor) {
461 RunTest(FILE_PATH_LITERAL("input-color.html"));
464 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
465 AccessibilityInputImageButtonInMenu) {
466 RunTest(FILE_PATH_LITERAL("input-image-button-in-menu.html"));
469 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityInputRange) {
470 RunTest(FILE_PATH_LITERAL("input-range.html"));
473 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
474 AccessibilityInputTextNameCalc) {
475 RunTest(FILE_PATH_LITERAL("input-text-name-calc.html"));
478 // crbug.com/98976 will cause new elements to be added to the Blink a11y tree
479 // Re-baseline after the Blink change goes in
480 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
481 DISABLED_AccessibilityInputTypes) {
482 RunTest(FILE_PATH_LITERAL("input-types.html"));
485 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityLabel) {
486 RunTest(FILE_PATH_LITERAL("label.html"));
489 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityLandmark) {
490 RunTest(FILE_PATH_LITERAL("landmark.html"));
493 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityListMarkers) {
494 RunTest(FILE_PATH_LITERAL("list-markers.html"));
497 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
498 AccessibilityModalDialogClosed) {
499 RunTest(FILE_PATH_LITERAL("modal-dialog-closed.html"));
502 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
503 AccessibilityModalDialogOpened) {
504 RunTest(FILE_PATH_LITERAL("modal-dialog-opened.html"));
507 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
508 AccessibilityModalDialogInIframeClosed) {
509 RunTest(FILE_PATH_LITERAL("modal-dialog-in-iframe-closed.html"));
512 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
513 AccessibilityModalDialogInIframeOpened) {
514 RunTest(FILE_PATH_LITERAL("modal-dialog-in-iframe-opened.html"));
517 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
518 AccessibilityModalDialogStack) {
519 RunTest(FILE_PATH_LITERAL("modal-dialog-stack.html"));
522 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityP) {
523 RunTest(FILE_PATH_LITERAL("p.html"));
526 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityRegion) {
527 RunTest(FILE_PATH_LITERAL("region.html"));
530 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilitySelect) {
531 RunTest(FILE_PATH_LITERAL("select.html"));
534 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilitySpan) {
535 RunTest(FILE_PATH_LITERAL("span.html"));
538 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilitySpinButton) {
539 RunTest(FILE_PATH_LITERAL("spinbutton.html"));
542 // TODO(dmazzoni): Rebaseline this test after Blink rolls past r155083.
543 // See http://crbug.com/265619
544 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, DISABLED_AccessibilitySvg) {
545 RunTest(FILE_PATH_LITERAL("svg.html"));
548 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityTab) {
549 RunTest(FILE_PATH_LITERAL("tab.html"));
552 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityTableSimple) {
553 RunTest(FILE_PATH_LITERAL("table-simple.html"));
556 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityTableSpans) {
557 RunTest(FILE_PATH_LITERAL("table-spans.html"));
560 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityTransition) {
561 RunTest(FILE_PATH_LITERAL("transition.html"));
564 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
565 AccessibilityToggleButton) {
566 RunTest(FILE_PATH_LITERAL("togglebutton.html"));
569 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityUl) {
570 RunTest(FILE_PATH_LITERAL("ul.html"));
573 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest, AccessibilityWbr) {
574 RunTest(FILE_PATH_LITERAL("wbr.html"));
577 IN_PROC_BROWSER_TEST_F(DumpAccessibilityTreeTest,
578 AccessibilityAriaActivedescendant) {
579 RunTest(FILE_PATH_LITERAL("aria-activedescendant.html"));
582 } // namespace content