From 23c31dd8c447db2d0f44ce6137edeb0ce3c939a1 Mon Sep 17 00:00:00 2001 From: Valerie Young Date: Tue, 30 Apr 2024 17:18:04 -0700 Subject: [PATCH 01/38] Create new test type for accessibility API testing --- .taskcluster.yml | 2 +- lint.ignore | 1 + tools/ci/tc/tasks/test.yml | 2 +- tools/docker/Dockerfile | 3 + tools/manifest/item.py | 16 ++ tools/manifest/manifest.py | 4 +- tools/manifest/sourcefile.py | 27 +- tools/mypy.ini | 9 + tools/wpt/run.py | 4 + tools/wpt/testfiles.py | 2 +- .../requirements_platform_accessibility.txt | 8 + tools/wptrunner/wptrunner/browsers/chrome.py | 16 +- .../wptrunner/wptrunner/browsers/chromium.py | 1 + tools/wptrunner/wptrunner/browsers/edge.py | 3 + tools/wptrunner/wptrunner/browsers/firefox.py | 17 +- tools/wptrunner/wptrunner/browsers/servo.py | 1 + tools/wptrunner/wptrunner/executors/base.py | 2 +- .../executors/pytestrunner/runner.py | 1 + tools/wptrunner/wptrunner/wpttest.py | 23 +- wai-aria-aam/__init__.py | 0 .../attribute/aria-autocomplete_tentative.py | 33 +++ .../attribute/aria-braillelabel_tentative.py | 37 +++ .../attribute/aria-errormessage_tentative.py | 51 ++++ wai-aria-aam/conftest.py | 11 + wai-aria-aam/role/__init__.py | 0 wai-aria-aam/role/blockquote_tentative.py | 44 ++++ ...ria-pressed_and_aria-haspopup_tentative.py | 46 ++++ ...efined_value_for_aria-pressed_tentative.py | 43 ++++ ...false_value_for_aria-haspopup_tentative.py | 46 ++++ wai-aria-aam/support/__init__.py | 0 wai-aria-aam/support/atspi_wrapper.py | 236 ++++++++++++++++++ wai-aria-aam/support/axapi_wrapper.py | 128 ++++++++++ wai-aria-aam/support/fixtures_a11y_api.py | 54 ++++ wai-aria-aam/support/ia2/constants.py | 208 +++++++++++++++ wai-aria-aam/support/ia2/ia2_api_all.tlb | Bin 0 -> 29016 bytes wai-aria-aam/support/ia2_wrapper.py | 165 ++++++++++++ 36 files changed, 1227 insertions(+), 17 deletions(-) create mode 100644 tools/wptrunner/requirements_platform_accessibility.txt create mode 100644 wai-aria-aam/__init__.py create mode 100644 wai-aria-aam/attribute/aria-autocomplete_tentative.py create mode 100644 wai-aria-aam/attribute/aria-braillelabel_tentative.py create mode 100644 wai-aria-aam/attribute/aria-errormessage_tentative.py create mode 100644 wai-aria-aam/conftest.py create mode 100644 wai-aria-aam/role/__init__.py create mode 100644 wai-aria-aam/role/blockquote_tentative.py create mode 100644 wai-aria-aam/role/button_with_default_values_for_aria-pressed_and_aria-haspopup_tentative.py create mode 100644 wai-aria-aam/role/button_with_defined_value_for_aria-pressed_tentative.py create mode 100644 wai-aria-aam/role/button_with_non_false_value_for_aria-haspopup_tentative.py create mode 100644 wai-aria-aam/support/__init__.py create mode 100644 wai-aria-aam/support/atspi_wrapper.py create mode 100644 wai-aria-aam/support/axapi_wrapper.py create mode 100644 wai-aria-aam/support/fixtures_a11y_api.py create mode 100644 wai-aria-aam/support/ia2/constants.py create mode 100644 wai-aria-aam/support/ia2/ia2_api_all.tlb create mode 100644 wai-aria-aam/support/ia2_wrapper.py diff --git a/.taskcluster.yml b/.taskcluster.yml index 5bef41d7d97406..d120d54d3960f1 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -57,7 +57,7 @@ tasks: owner: ${owner} source: ${event.repository.clone_url} payload: - image: ghcr.io/web-platform-tests/wpt:2 + image: ghcr.io/web-platform-tests/wpt:3 maxRunTime: 7200 artifacts: public/results: diff --git a/lint.ignore b/lint.ignore index 408729b9df62f0..7b21a36c4585f5 100644 --- a/lint.ignore +++ b/lint.ignore @@ -52,6 +52,7 @@ TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.wbn TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.avif TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.annexb TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.crx +TRAILING WHITESPACE, INDENT TABS, CR AT EOL: *.tlb ## .gitignore W3C-TEST.ORG: .gitignore diff --git a/tools/ci/tc/tasks/test.yml b/tools/ci/tc/tasks/test.yml index 8973d3beb49f9e..ef6d3f021fa8a0 100644 --- a/tools/ci/tc/tasks/test.yml +++ b/tools/ci/tc/tasks/test.yml @@ -4,7 +4,7 @@ components: workerType: ci schedulerId: taskcluster-github deadline: "24 hours" - image: ghcr.io/web-platform-tests/wpt:2 + image: ghcr.io/web-platform-tests/wpt:3 maxRunTime: 7200 artifacts: public/results: diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index a936ad9167f058..53f1c9f3d7a5be 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -20,6 +20,9 @@ RUN apt-get -qqy update \ glib-networking-services \ gstreamer1.0-plugins-bad \ gstreamer1.0-gl \ + libatspi2.0-dev \ + libcairo2-dev \ + libgirepository1.0-dev \ libosmesa6-dev \ libproxy1-plugin-webkit \ libvirt-daemon-system \ diff --git a/tools/manifest/item.py b/tools/manifest/item.py index 6aa9f6c8e68347..ec3a28c8d3c78d 100644 --- a/tools/manifest/item.py +++ b/tools/manifest/item.py @@ -360,6 +360,22 @@ def to_json(self) -> Tuple[Optional[Text], Dict[Text, Any]]: return rv +class AccessibilityAPIMappingTest(URLManifestItem): + __slots__ = () + + item_type = "aamtest" + + @property + def timeout(self) -> Optional[Text]: + return self._extras.get("timeout") + + def to_json(self) -> Tuple[Optional[Text], Dict[Text, Any]]: + rv = super().to_json() + if self.timeout is not None: + rv[-1]["timeout"] = self.timeout + return rv + + class SupportFile(ManifestItem): __slots__ = () diff --git a/tools/manifest/manifest.py b/tools/manifest/manifest.py index 12260192c3297c..9cc04785836ebf 100644 --- a/tools/manifest/manifest.py +++ b/tools/manifest/manifest.py @@ -8,7 +8,8 @@ from . import jsonlib from . import vcs -from .item import (ConformanceCheckerTest, +from .item import (AccessibilityAPIMappingTest, + ConformanceCheckerTest, CrashTest, ManifestItem, ManualTest, @@ -47,6 +48,7 @@ class InvalidCacheError(Exception): "crashtest": CrashTest, "manual": ManualTest, "wdspec": WebDriverSpecTest, + "aamtest": AccessibilityAPIMappingTest, "conformancechecker": ConformanceCheckerTest, "visual": VisualTest, "spec": SpecItem, diff --git a/tools/manifest/sourcefile.py b/tools/manifest/sourcefile.py index 101f28dbdb510a..76bb151cc8028b 100644 --- a/tools/manifest/sourcefile.py +++ b/tools/manifest/sourcefile.py @@ -16,7 +16,8 @@ import html5lib from . import XMLParser -from .item import (ConformanceCheckerTest, +from .item import (AccessibilityAPIMappingTest, + ConformanceCheckerTest, CrashTest, ManifestItem, ManualTest, @@ -401,6 +402,15 @@ def name_is_webdriver(self) -> bool: self.filename not in ("__init__.py", "conftest.py") and fnmatch(self.filename, wd_pattern)) + @property + def name_is_wai_aria_aam(self) -> bool: + """Check if the file name matches the conditions for the file to + be a WAI-ARIA AAM spec test file""" + rel_path_parts = self.rel_path_parts + return ((rel_path_parts[0] == "wai-aria-aam" and len(rel_path_parts) > 1) and + self.filename not in ("__init__.py", "conftest.py") and + fnmatch(self.filename, wd_pattern)) + @property def name_is_reference(self) -> bool: """Check if the file name matches the conditions for the file to @@ -492,7 +502,7 @@ def test262_test_record(self) -> Optional[test262.TestRecord]: def script_metadata(self) -> Optional[List[Tuple[Text, Text]]]: if self.name_is_worker or self.name_is_multi_global or self.name_is_window or self.name_is_extension: regexp = js_meta_re - elif self.name_is_webdriver: + elif self.name_is_webdriver or self.name_is_wai_aria_aam: regexp = python_meta_re elif self.name_is_test262: if self.test262_test_record is None: @@ -931,6 +941,9 @@ def possible_types(self) -> Set[Text]: if self.name_is_webdriver: return {WebDriverSpecTest.item_type} + if self.name_is_wai_aria_aam: + return {AccessibilityAPIMappingTest.item_type} + if self.name_is_visual: return {VisualTest.item_type} @@ -1021,6 +1034,16 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]: timeout=self.timeout )] + elif self.name_is_wai_aria_aam: + rv = AccessibilityAPIMappingTest.item_type, [ + AccessibilityAPIMappingTest( + self.tests_root, + self.rel_path, + self.url_base, + self.rel_url, + timeout=self.timeout + )] + elif self.name_is_visual: rv = VisualTest.item_type, [ VisualTest( diff --git a/tools/mypy.ini b/tools/mypy.ini index c7376bc4f144af..d39878b0b0c2bc 100644 --- a/tools/mypy.ini +++ b/tools/mypy.ini @@ -42,6 +42,9 @@ follow_imports = silent follow_imports = silent # Ignore missing or untyped libraries. +[mypy-ApplicationServices.*] +ignore_missing_imports = True + [mypy-Cocoa.*] ignore_missing_imports = True @@ -51,9 +54,15 @@ ignore_missing_imports = True [mypy-Quartz.*] ignore_missing_imports = True +[mypy-comtypes.*] +ignore_missing_imports = True + [mypy-dnslib.*] ignore_missing_imports = True +[mypy-gi.*] +ignore_missing_imports = True + [mypy-marionette_driver.*] ignore_missing_imports = True diff --git a/tools/wpt/run.py b/tools/wpt/run.py index fc51c67e9bf1ac..10f6a2156fd566 100644 --- a/tools/wpt/run.py +++ b/tools/wpt/run.py @@ -947,6 +947,10 @@ def setup_wptrunner(venv, **kwargs): if not venv.skip_virtualenv_setup: requirements = [os.path.join(wpt_root, "tools", "wptrunner", "requirements.txt")] requirements.extend(setup_cls.requirements()) + + if "aamtest" in kwargs["test_types"]: + requirements.append(os.path.join(wpt_root, "tools", "wptrunner", "requirements_platform_accessibility.txt")) + venv.install_requirements(*requirements) affected_revish = kwargs.get("affected") diff --git a/tools/wpt/testfiles.py b/tools/wpt/testfiles.py index 604014855f1caa..3a070057e13280 100644 --- a/tools/wpt/testfiles.py +++ b/tools/wpt/testfiles.py @@ -238,7 +238,7 @@ def affected_testfiles(files_changed: Iterable[Text], nontests_changed = set(files_changed) wpt_manifest = load_manifest(manifest_path, manifest_update) - test_types = ["crashtest", "print-reftest", "reftest", "test262", "testharness", "wdspec"] + test_types = ["crashtest", "print-reftest", "reftest", "test262", "testharness", "wdspec", "aamtest"] support_files = {os.path.join(wpt_root, path) for _, path, _ in wpt_manifest.itertypes("support")} wdspec_test_files = {os.path.join(wpt_root, path) diff --git a/tools/wptrunner/requirements_platform_accessibility.txt b/tools/wptrunner/requirements_platform_accessibility.txt new file mode 100644 index 00000000000000..b14331f7e25e93 --- /dev/null +++ b/tools/wptrunner/requirements_platform_accessibility.txt @@ -0,0 +1,8 @@ +comtypes==1.4.11; sys_platform == 'win32' +PyGObject==3.50.1; sys_platform == 'linux' and python_version >= '3.9' +PyGObject==3.48.2; sys_platform == 'linux' and python_version < '3.9' +pyobjc-framework-Accessibility==11.1; sys_platform == 'darwin' and python_version >= '3.9' +pyobjc-framework-ApplicationServices==11.1; sys_platform == 'darwin' and python_version >= '3.9' +pyobjc==10.3.2; sys_platform == 'darwin' and python_version < '3.9' +pyobjc-framework-Accessibility==10.3.2; sys_platform == 'darwin' and python_version < '3.9' +pyobjc-framework-ApplicationServices==10.3.2; sys_platform == 'darwin' and python_version < '3.9' diff --git a/tools/wptrunner/wptrunner/browsers/chrome.py b/tools/wptrunner/wptrunner/browsers/chrome.py index 6f7be4eebdbff5..79366784a5adf4 100644 --- a/tools/wptrunner/wptrunner/browsers/chrome.py +++ b/tools/wptrunner/wptrunner/browsers/chrome.py @@ -1,5 +1,6 @@ # mypy: allow-untyped-defs +import os import re import time @@ -33,6 +34,7 @@ "reftest": "ChromeDriverRefTestExecutor", "print-reftest": "ChromeDriverPrintRefTestExecutor", "wdspec": "WdspecExecutor", + "aamtest": "WdspecExecutor", "crashtest": "ChromeDriverCrashTestExecutor", "test262": "ChromeDriverTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", @@ -225,6 +227,13 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, subsuite if test_type == "wdspec": executor_kwargs["binary_args"] = chrome_options["args"] + if test_type == "aamtest": + # Necessary to force chrome to register in AT-SPI2. + os.environ["ACCESSIBILITY_ENABLED"] = "1" + if "--force-renderer-accessibility" not in chrome_options["args"]: + chrome_options["args"].append("--force-renderer-accessibility") + executor_kwargs["binary_args"] = chrome_options["args"] + executor_kwargs["capabilities"] = capabilities return executor_kwargs @@ -259,9 +268,10 @@ def __init__(self, self._require_webdriver_bidi: Optional[bool] = None def restart_on_test_type_change(self, new_test_type: str, old_test_type: str) -> bool: - # Restart the test runner when switch from/to wdspec tests. Wdspec test - # is using a different protocol class so a restart is always needed. - if "wdspec" in [old_test_type, new_test_type]: + # Restart the test runner when switch from/to wdspec or aamtest tests. + # These tests use a different protocol class so a restart is always needed. + wdspec_types = {"wdspec", "aamtest"} + if old_test_type in wdspec_types or new_test_type in wdspec_types: return True return False diff --git a/tools/wptrunner/wptrunner/browsers/chromium.py b/tools/wptrunner/wptrunner/browsers/chromium.py index a459835a6d1a96..89abb330099a67 100644 --- a/tools/wptrunner/wptrunner/browsers/chromium.py +++ b/tools/wptrunner/wptrunner/browsers/chromium.py @@ -17,6 +17,7 @@ "reftest": "WebDriverRefTestExecutor", "print-reftest": "ChromeDriverPrintRefTestExecutor", "wdspec": "WdspecExecutor", + "aamtest": "WdspecExecutor", "crashtest": "WebDriverCrashtestExecutor", "test262": "WebDriverTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", diff --git a/tools/wptrunner/wptrunner/browsers/edge.py b/tools/wptrunner/wptrunner/browsers/edge.py index 6145916f626096..913058a673ef41 100644 --- a/tools/wptrunner/wptrunner/browsers/edge.py +++ b/tools/wptrunner/wptrunner/browsers/edge.py @@ -19,8 +19,11 @@ "reftest": "EdgeDriverRefTestExecutor", "print-reftest": "EdgeDriverPrintRefTestExecutor", "wdspec": "WdspecExecutor", + "aamtest": "WdspecExecutor", "crashtest": "WebDriverCrashtestExecutor", "test262": "EdgeDriverTestharnessExecutor"}, + "crashtest": "WebDriverCrashtestExecutor"}, + "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_extras": "env_extras", diff --git a/tools/wptrunner/wptrunner/browsers/firefox.py b/tools/wptrunner/wptrunner/browsers/firefox.py index 609afc02e253ca..93f28473d43144 100644 --- a/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/tools/wptrunner/wptrunner/browsers/firefox.py @@ -40,12 +40,14 @@ __wptrunner__ = {"product": "firefox", "check_args": "check_args", "browser": {None: "FirefoxBrowser", - "wdspec": "FirefoxWdSpecBrowser"}, + "wdspec": "FirefoxWdSpecBrowser", + "aamtest": "FirefoxWdSpecBrowser"}, "executor": {"crashtest": "MarionetteCrashtestExecutor", "testharness": "MarionetteTestharnessExecutor", "reftest": "MarionetteRefTestExecutor", "print-reftest": "MarionettePrintRefTestExecutor", "wdspec": "MarionetteWdspecExecutor", + "aamtest": "MarionetteWdspecExecutor", "test262": "MarionetteTestharnessExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", @@ -72,7 +74,7 @@ def get_timeout_multiplier(test_type, run_info_data, **kwargs): return 4 * multiplier else: return 2 * multiplier - elif test_type == "wdspec": + elif test_type in ("wdspec", "aamtest"): if (run_info_data.get("asan") or run_info_data.get("ccov") or run_info_data.get("debug")): @@ -127,7 +129,7 @@ def browser_kwargs(logger, test_type, run_info_data, config, subsuite, **kwargs) "gmp_path": kwargs["gmp_path"] if "gmp_path" in kwargs else None, "debug_test": kwargs["debug_test"]} - if test_type == "wdspec": + if test_type in ("wdspec", "aamtest"): browser_kwargs["webdriver_binary"] = kwargs["webdriver_binary"] browser_kwargs["webdriver_args"] = kwargs["webdriver_args"].copy() @@ -146,6 +148,9 @@ def browser_kwargs(logger, test_type, run_info_data, config, subsuite, **kwargs) browser_kwargs["test_type"] = test_type browser_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type, run_info_data, **kwargs) + if test_type == "aamtest" and ('accessibility.force_disabled', '-1') not in browser_kwargs["extra_prefs"]: + browser_kwargs["extra_prefs"].append(('accessibility.force_disabled', '-1')) + browser_kwargs["extra_prefs"].extend(subsuite.config.get("prefs", [])) return browser_kwargs @@ -173,7 +178,8 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, else: cache_screenshots = major_version < 14 executor_kwargs["cache_screenshots"] = cache_screenshots - if test_type == "wdspec": + + if test_type in ("wdspec", "aamtest"): options = {"args": []} if kwargs["binary"]: executor_kwargs["webdriver_args"].extend(["--binary", kwargs["binary"]]) @@ -773,7 +779,7 @@ def _get_default_prefs(self): if self.test_type == "print-reftest": prefs["print.always_print_silent"] = True - if self.test_type == "wdspec": + if self.test_type in ("wdspec", "aamtest"): prefs.update( { "remote.prefs.recommended": True, @@ -976,6 +982,7 @@ def __init__(self, logger, binary, package_name, prefs_root, webdriver_binary, w self.env = self.get_env(binary, debug_info, headless, gmp_path, chaos_mode_flags, e10s) + # Todo: need test type to use "aam" test in profile_creator_cls profile_creator = profile_creator_cls(logger, prefs_root, config, diff --git a/tools/wptrunner/wptrunner/browsers/servo.py b/tools/wptrunner/wptrunner/browsers/servo.py index 950a921ac7d2f5..33707c04aeb381 100644 --- a/tools/wptrunner/wptrunner/browsers/servo.py +++ b/tools/wptrunner/wptrunner/browsers/servo.py @@ -27,6 +27,7 @@ "reftest": "ServoRefTestExecutor", "crashtest": "ServoCrashtestExecutor", "wdspec": "WdspecExecutor", + "aamtest": "WdspecExecutor", "test262": "ServoTestharnessExecutor", }, "browser_kwargs": "browser_kwargs", diff --git a/tools/wptrunner/wptrunner/executors/base.py b/tools/wptrunner/wptrunner/executors/base.py index fc4cb1abfaa301..b784c1669f6a3c 100644 --- a/tools/wptrunner/wptrunner/executors/base.py +++ b/tools/wptrunner/wptrunner/executors/base.py @@ -47,7 +47,7 @@ def executor_kwargs(test_type, test_environment, run_info_data, subsuite, **kwar executor_kwargs["screenshot_cache"] = screenshot_cache executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"] - if test_type == "wdspec": + if test_type in ("wdspec", "aamtest"): executor_kwargs["binary"] = kwargs["binary"] executor_kwargs["binary_args"] = kwargs["binary_args"].copy() executor_kwargs["webdriver_binary"] = kwargs["webdriver_binary"] diff --git a/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py b/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py index df7b7c40132267..57eab56082dfa5 100644 --- a/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py +++ b/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py @@ -52,6 +52,7 @@ def run(path, server_config, session_config, timeout=0): config = session_config.copy() config["wptserve"] = server_config.as_dict() + config["timeout"] = timeout with open(config_path, "w") as f: json.dump(config, f) diff --git a/tools/wptrunner/wptrunner/wpttest.py b/tools/wptrunner/wptrunner/wpttest.py index b05adaff2f89a5..ed879c22df99be 100644 --- a/tools/wptrunner/wptrunner/wpttest.py +++ b/tools/wptrunner/wptrunner/wpttest.py @@ -10,8 +10,7 @@ from .wptmanifest.parser import atoms atom_reset = atoms["Reset"] -enabled_tests = {"testharness", "reftest", "wdspec", "crashtest", "print-reftest", "test262"} - +enabled_tests = {"testharness", "reftest", "wdspec", "crashtest", "print-reftest", "test262", "aamtest"} class Result(ABC): default_expected: ClassVar[str] @@ -78,6 +77,16 @@ class WdspecSubtestResult(SubtestResult): statuses = {"PASS", "FAIL", "ERROR"} +class AamSpecResult(Result): + default_expected = "OK" + statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"} + + +class AamSpecSubtestResult(SubtestResult): + default_expected = "PASS" + statuses = {"PASS", "FAIL", "ERROR"} + + class CrashtestResult(Result): default_expected = "PASS" statuses = {"PASS", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", @@ -736,6 +745,15 @@ class WdspecTest(Test): long_timeout = 180 # 3 minutes +class AamSpecTest(Test): + result_cls = AamSpecResult + subtest_result_cls = AamSpecSubtestResult + test_type = "aamtest" + + default_timeout = 25 + long_timeout = 180 # 3 minutes + + class CrashTest(Test): result_cls = CrashtestResult test_type = "crashtest" @@ -765,6 +783,7 @@ def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_meta "print-reftest": PrintReftestTest, "testharness": TestharnessTest, "wdspec": WdspecTest, + "aamtest": AamSpecTest, "crashtest": CrashTest, "test262": Test262Test} diff --git a/wai-aria-aam/__init__.py b/wai-aria-aam/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/wai-aria-aam/attribute/aria-autocomplete_tentative.py b/wai-aria-aam/attribute/aria-autocomplete_tentative.py new file mode 100644 index 00000000000000..eb87d2db6d5166 --- /dev/null +++ b/wai-aria-aam/attribute/aria-autocomplete_tentative.py @@ -0,0 +1,33 @@ +import pytest + +TEST_HTML = { + "both": "", + "inline": "", + "list": "", +} + +@pytest.mark.parametrize("test_name", TEST_HTML.keys()) +def test_atspi(atspi, session, inline, test_name): + if not atspi: + return + + session.url = inline(TEST_HTML[test_name]) + + node = atspi.find_node("test", session.url) + assert f"autocomplete:{test_name}" in atspi.Accessible.get_attributes_as_array(node) + assert "STATE_SUPPORTS_AUTOCOMPLETION" in atspi.get_state_list_helper(node) + + +# Intentionally no AX API test, AX API does not map. + +@pytest.mark.parametrize("test_name", TEST_HTML.keys()) +def test_ia2(ia2, session, inline, test_name): + if not ia2: + return + + session.url = inline(TEST_HTML[test_name]) + + # Todo: Add test for IA2. + + +# Intentionally no UIA test, UIA does not map. diff --git a/wai-aria-aam/attribute/aria-braillelabel_tentative.py b/wai-aria-aam/attribute/aria-braillelabel_tentative.py new file mode 100644 index 00000000000000..26031b46554b01 --- /dev/null +++ b/wai-aria-aam/attribute/aria-braillelabel_tentative.py @@ -0,0 +1,37 @@ + +TEST_HTML = '" def test_atspi(atspi, session, inline): session.url = inline(TEST_HTML) diff --git a/core-aam/aamtests/support/axapi_wrapper.py b/core-aam/aamtests/support/axapi_wrapper.py index 6bfeb408fbd949..d891ec7ca03c97 100644 --- a/core-aam/aamtests/support/axapi_wrapper.py +++ b/core-aam/aamtests/support/axapi_wrapper.py @@ -87,6 +87,8 @@ def _find_tab(self) -> Optional[AXUIElement]: if err: continue if role == "AXWebArea": + # TODO: AtspiWrapper will check that the found tab is the correct + # tab by checking the URL. Perform this check here. return node err, children = AXUIElementCopyAttributeValue(node, "AXChildren", None) diff --git a/core-aam/aamtests/support/fixtures_a11y_api.py b/core-aam/aamtests/support/fixtures_a11y_api.py index e3e356ba93a44f..de64518d0d65fe 100644 --- a/core-aam/aamtests/support/fixtures_a11y_api.py +++ b/core-aam/aamtests/support/fixtures_a11y_api.py @@ -3,6 +3,7 @@ def pid_from(capabilities): + # TODO: add support for Edge, Safari. if capabilities["browserName"] == "chrome": return capabilities["goog:processID"], "chrome" if capabilities["browserName"] == "firefox": From a2d6005d1cff5a03364692709def37a9e938b588 Mon Sep 17 00:00:00 2001 From: Valerie Young Date: Mon, 27 Apr 2026 14:32:07 -0700 Subject: [PATCH 35/38] Add documentation for aamtest test type (#58632) --- docs/test-suite-design.md | 4 ++ docs/writing-tests/aamtest.md | 132 ++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 docs/writing-tests/aamtest.md diff --git a/docs/test-suite-design.md b/docs/test-suite-design.md index 6a104e2f1d42fd..c9d48c015f55ae 100644 --- a/docs/test-suite-design.md +++ b/docs/test-suite-design.md @@ -68,6 +68,10 @@ expectations: * [wdspec][] tests are written in Python and test [the WebDriver browser automation protocol](https://w3c.github.io/webdriver/) +* [aamtest][] tests are written in Python and test accessibility API mappings, + such as [Core Accessibility API Mappings](https://w3c.github.io/core-aam/) + or [HTML Accessibility API Mappings](https://w3c.github.io/html-aam/). + * [Manual tests][manual] rely on a human to run them and determine their result. diff --git a/docs/writing-tests/aamtest.md b/docs/writing-tests/aamtest.md new file mode 100644 index 00000000000000..58214e3e001aeb --- /dev/null +++ b/docs/writing-tests/aamtest.md @@ -0,0 +1,132 @@ +# aamtest tests + +The aamtest tests are used to verify the mapping of web content to +browser-exposed platform-specific accessibility APIs. These mappings are +specified by the ARIA working group of the W3C in the following specifications: + +* [Core-AAM](https://w3c.github.io/core-aam) +* [HTML-AAM](https://w3c.github.io/html-aam) +* [DPUB-AAM](https://w3c.github.io/dpub-aam) +* [MathML-AAM](https://w3c.github.io/mathml-aam) +* [Graphics-AAM](https://w3c.github.io/graphics-aam) +* [SVG-AAM](https://w3c.github.io/svg-aam/) + +These tests are written in [the Python programming +language](https://www.python.org/) and structured with [the pytest testing +framework](https://docs.pytest.org/en/latest/). + +The aamtest type is built on the [wdspec](wdspec) test type, and has access to +all the python fixtures defined for wdspec tests. It uses the web-platform-tests +maintained WebDriver client library to load HTML and to send other WebDriver +commands to the browser. + +The `wptrunner` will know a python file is an aamtest if it is contained within +an `aamtests` directory. + +## Platform-Specific Accessibility APIs + +Accessibility APIs are platform specific, each platform has their own API +(sometimes more than one). Assistive technologies, such as screen readers, +interact with the browser on behalf of a user via these APIs. The AAM +specifications explain how to expose web content through these APIs. You can +read more about [the APIs in +Core-AAM](https://w3c.github.io/core-aam/#intro_aapi). + +The table below lists: + +* APIs supported by the aamtest framework +* The name of the pytest fixture that returns access to the API (if you are + on the correct platform). +* The platform of that API. +* The python library that provides bindings to query the API. + +| API Name | Fixture Name | Platform | Python Bindings | +|---|---|---|---| +| Accessibility Toolkit ([ATK](https://developer.gnome.org/atk/stable/)) and Assistive Technology Service Provider Interface ([AT-SPI](https://gnome.pages.gitlab.gnome.org/at-spi2-core/libatspi/)) | `atspi` | Linux | [Provided through PyGObject](https://lazka.github.io/pgi-docs/#Atspi-2.0) | +| The NSAccessibility Protocol for macOS ([AX API](https://developer.apple.com/documentation/appkit/nsaccessibility)) | `axapi` | macOS | [pyobjc-framework-Accessibility](https://pypi.org/project/pyobjc-framework-Accessibility/) | +| MSAA with IAccessible2 1.3 ([IA2](https://wiki.linuxfoundation.org/accessibility/iaccessible2/start)) | `ia2` | Windows | Loading module [ia2_api_all.idl](https://github.com/LinuxA11y/IAccessible2) with [comtypes](https://pypi.org/project/comtypes/) | + +The APIs are exposed through a pytest fixture with the name in the table +above. The pytest fixture returns a wrapped version of the API. Requesting this +fixture on a platform where it is not supported will result in a `MISSING` +subtest. It is expected that each test file run on a given platform will have +one or more subtests that run to completion, as well as several subtests who's +results will not be recorded. This is because each test file should show how the +**same markup** is exposed in each supporting accessibility APIs/platforms. + +### Package dependencies for Linux API AT-SPI Python Bindings + +In order to test the Linux API AT-SPI, you need to have the following packages +installed: + +``` +sudo apt install libatspi2.0-dev libcairo2-dev libgirepository1.0-dev +``` + +## Adding new tests + +If you would like to add a new aamtest to a specification that does not yet have +coverage, add the tests to an `aamtests` subfolder. This subfolder indicates the +python files within it will be run as an aamtest. This includes restarting the +browser with accessibility enabled, if you are doing a full run of the test +suite. + +In the `aamtests` subfolder, the `conftest.py` will need to add the +`webdriver/test/support` path and `core-aam/aamtests/support` path to the +sys.path and add the paths as `pytest_plugins` in order to have access to the +appropriate fixtures. See `core-aam/aamtests/conftest.py`. + +### Test design + +Similar to [testharness.js](testharness) tests, each file is a test, and a +function that begins with the name "test_" is a subtest of that test. + +A typical test file contains some html markup and several subtests. Each subtest +will test how that markup is exposed in a single accessibility API. For example, +if you are testing how `
foobar widget
` is exposed in +accessibility APIs, and foobars are supported in the Linux API AT-SPI and macOS +API AX API, you should add a subtest called `test_atspi` and `test_axapi`, +respectively. Both subtests will load the same HTML, then query their respective +accessibility APIs. The subtest name should include, at least, the name of the +API being tested for ease of understanding the test results. + +For example, the file `foobar.py`: +```python +TEST_HTML = "
" + +# Test of the Linux accessibility API: AT-SPI +def test_atspi(atspi, session, inline): + # The `session` and `inline` fixtures are provided from the `wdspec` test infrastructure. + session.url = inline(TEST_HTML) + + # The `atspi` fixture wraps the AT-SPI python bindings and provides some helper functions, + # such as `find_node`, which finds a node by DOM ID. + node = atspi.find_node("test", session.url) + assert atspi.Accessible.get_role(node) == atspi.Role.FOOBAR + +# Test of the macOS accessibility API: AX API +def test_axapi(axapi, session, inline): + # The `session` and `inline` fixtures are provided from the `wdspec` test infrastructure. + session.url = inline(TEST_HTML) + + # The `axapi` fixture wraps the AX API python bindings and provides some helper functions, + # such as `find_node`, which finds a node by DOM ID. + node = axapi.find_node("test", session.url) + role = axapi.AXUIElementCopyAttributeValue(node, "AXRole", None)[1] + assert role == "AXFoobar" +``` + +## Adding support for an unsupported API + +To add an unsupported API: + +1. Add the python package that provides the python bindings for that API to + `tools/wptrunner/requirements_platform_accessibility.txt`. +2. Create a wrapper object for the new API in `newapi_wrapper.py` in + `core-aam/aamtests/support/`. It must inherit from `ApiWrapper` and follow + the same conventions as the other API wrappers, as appropriate. +3. Add the fixture for that API in + `core-aam/aamtests/support/fixtures_a11y_api.py`. +4. Update the table in the "Platform-Specific Accessibility APIs" section of this + document. +5. Add a new subtest to all the files that contain markup you would like to test. From b23715cec776e077782148a84a403cf7f9626d17 Mon Sep 17 00:00:00 2001 From: Valerie Young Date: Mon, 27 Apr 2026 16:35:23 -0700 Subject: [PATCH 36/38] Revert "Add back polling for firefox" This reverts commit f89b94990ed47fd060394ac9f7b63b529e03eb77. --- core-aam/aamtests/support/atspi_wrapper.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/core-aam/aamtests/support/atspi_wrapper.py b/core-aam/aamtests/support/atspi_wrapper.py index dc35c2954f9d0b..145b2b647ae3cd 100644 --- a/core-aam/aamtests/support/atspi_wrapper.py +++ b/core-aam/aamtests/support/atspi_wrapper.py @@ -29,13 +29,7 @@ def find_node(self, dom_id: str, url: str) -> Atspi.Accessible: self._find_fully_loaded_tab, f"Timeout looking for url: {self.test_url}" ) - # Polling for the node with ID because in Firefox the node's - # id attribute may be set after the tab is ready, leading to flakes. - test_node: Atspi.Accessible = self._poll_for( - lambda: self._find_node_by_id(self.document, dom_id), - f"Timout looking for node with id '{dom_id}' in accessibility API ATSPI.", - ) - + test_node = self._find_node_by_id(self.document, dom_id); if not test_node: raise Exception(f"Did not find node with id '{dom_id}' in accessibility API ATSPI.") From 04e9829785d63675f2afa509715136f34d65f09e Mon Sep 17 00:00:00 2001 From: Valerie Young Date: Tue, 28 Apr 2026 12:19:39 -0700 Subject: [PATCH 37/38] Update docs with suggestion from cookiecrook --- docs/writing-tests/aamtest.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/writing-tests/aamtest.md b/docs/writing-tests/aamtest.md index 58214e3e001aeb..2a1758f287f185 100644 --- a/docs/writing-tests/aamtest.md +++ b/docs/writing-tests/aamtest.md @@ -2,7 +2,7 @@ The aamtest tests are used to verify the mapping of web content to browser-exposed platform-specific accessibility APIs. These mappings are -specified by the ARIA working group of the W3C in the following specifications: +specified by working groups of the W3C in the following specifications: * [Core-AAM](https://w3c.github.io/core-aam) * [HTML-AAM](https://w3c.github.io/html-aam) @@ -16,20 +16,20 @@ language](https://www.python.org/) and structured with [the pytest testing framework](https://docs.pytest.org/en/latest/). The aamtest type is built on the [wdspec](wdspec) test type, and has access to -all the python fixtures defined for wdspec tests. It uses the web-platform-tests +all the Python fixtures defined for wdspec tests. It uses the web-platform-tests maintained WebDriver client library to load HTML and to send other WebDriver commands to the browser. -The `wptrunner` will know a python file is an aamtest if it is contained within +The `wptrunner` will know a Python file is an aamtest if it is contained within an `aamtests` directory. ## Platform-Specific Accessibility APIs -Accessibility APIs are platform specific, each platform has their own API -(sometimes more than one). Assistive technologies, such as screen readers, -interact with the browser on behalf of a user via these APIs. The AAM -specifications explain how to expose web content through these APIs. You can -read more about [the APIs in +Accessibility APIs are platform (or operating system) specific, each platform +has their own API (sometimes more than one). Assistive technologies, such as +screen readers, interact with the browser on behalf of a user via these +APIs. The AAM specifications explain how to expose web content through these +APIs. You can read more about [the APIs in Core-AAM](https://w3c.github.io/core-aam/#intro_aapi). The table below lists: @@ -38,7 +38,7 @@ The table below lists: * The name of the pytest fixture that returns access to the API (if you are on the correct platform). * The platform of that API. -* The python library that provides bindings to query the API. +* The Python library that provides bindings to query the API. | API Name | Fixture Name | Platform | Python Bindings | |---|---|---|---| @@ -67,7 +67,7 @@ sudo apt install libatspi2.0-dev libcairo2-dev libgirepository1.0-dev If you would like to add a new aamtest to a specification that does not yet have coverage, add the tests to an `aamtests` subfolder. This subfolder indicates the -python files within it will be run as an aamtest. This includes restarting the +Python files within it will be run as an aamtest. This includes restarting the browser with accessibility enabled, if you are doing a full run of the test suite. @@ -99,7 +99,7 @@ def test_atspi(atspi, session, inline): # The `session` and `inline` fixtures are provided from the `wdspec` test infrastructure. session.url = inline(TEST_HTML) - # The `atspi` fixture wraps the AT-SPI python bindings and provides some helper functions, + # The `atspi` fixture wraps the AT-SPI Python bindings and provides some helper functions, # such as `find_node`, which finds a node by DOM ID. node = atspi.find_node("test", session.url) assert atspi.Accessible.get_role(node) == atspi.Role.FOOBAR @@ -109,7 +109,7 @@ def test_axapi(axapi, session, inline): # The `session` and `inline` fixtures are provided from the `wdspec` test infrastructure. session.url = inline(TEST_HTML) - # The `axapi` fixture wraps the AX API python bindings and provides some helper functions, + # The `axapi` fixture wraps the AX API Python bindings and provides some helper functions, # such as `find_node`, which finds a node by DOM ID. node = axapi.find_node("test", session.url) role = axapi.AXUIElementCopyAttributeValue(node, "AXRole", None)[1] @@ -120,7 +120,7 @@ def test_axapi(axapi, session, inline): To add an unsupported API: -1. Add the python package that provides the python bindings for that API to +1. Add the Python package that provides the Python bindings for that API to `tools/wptrunner/requirements_platform_accessibility.txt`. 2. Create a wrapper object for the new API in `newapi_wrapper.py` in `core-aam/aamtests/support/`. It must inherit from `ApiWrapper` and follow From d4fca40184b47ca789ecbac8c0244035ece1f58c Mon Sep 17 00:00:00 2001 From: Valerie Young Date: Tue, 28 Apr 2026 13:04:03 -0700 Subject: [PATCH 38/38] Fix tables and links --- docs/test-suite-design.md | 1 + docs/writing-tests/aamtest.md | 36 +++++++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/test-suite-design.md b/docs/test-suite-design.md index c9d48c015f55ae..1043ca62335c38 100644 --- a/docs/test-suite-design.md +++ b/docs/test-suite-design.md @@ -81,3 +81,4 @@ expectations: [manual]: writing-tests/manual [running-from-local-system]: running-tests/from-local-system [wdspec]: writing-tests/wdspec +[aamtest]: writing-tests/aamtest diff --git a/docs/writing-tests/aamtest.md b/docs/writing-tests/aamtest.md index 2a1758f287f185..273dff7c9f96e0 100644 --- a/docs/writing-tests/aamtest.md +++ b/docs/writing-tests/aamtest.md @@ -34,17 +34,33 @@ Core-AAM](https://w3c.github.io/core-aam/#intro_aapi). The table below lists: -* APIs supported by the aamtest framework -* The name of the pytest fixture that returns access to the API (if you are +* **API Name**: APIs supported by the aamtest framework +* **Fixture Name**: The name of the pytest fixture that returns access to the API (if you are on the correct platform). -* The platform of that API. -* The Python library that provides bindings to query the API. - -| API Name | Fixture Name | Platform | Python Bindings | -|---|---|---|---| -| Accessibility Toolkit ([ATK](https://developer.gnome.org/atk/stable/)) and Assistive Technology Service Provider Interface ([AT-SPI](https://gnome.pages.gitlab.gnome.org/at-spi2-core/libatspi/)) | `atspi` | Linux | [Provided through PyGObject](https://lazka.github.io/pgi-docs/#Atspi-2.0) | -| The NSAccessibility Protocol for macOS ([AX API](https://developer.apple.com/documentation/appkit/nsaccessibility)) | `axapi` | macOS | [pyobjc-framework-Accessibility](https://pypi.org/project/pyobjc-framework-Accessibility/) | -| MSAA with IAccessible2 1.3 ([IA2](https://wiki.linuxfoundation.org/accessibility/iaccessible2/start)) | `ia2` | Windows | Loading module [ia2_api_all.idl](https://github.com/LinuxA11y/IAccessible2) with [comtypes](https://pypi.org/project/comtypes/) | +* **Platform**: The platform of that API. +* **Python Bindings**: The Python library that provides bindings to query the API. + +```eval_rst +.. list-table:: + :header-rows: 1 + + * - API Name + - Fixture Name + - Platform + - Python Bindings + * - Accessibility Toolkit (`ATK `_) and Assistive Technology Service Provider Interface (`AT-SPI `_) + - ``atspi`` + - Linux + - `Provided through PyGObject `_ + * - The NSAccessibility Protocol for macOS (`AX API `_) + - ``axapi`` + - macOS + - `pyobjc-framework-Accessibility `_ + * - MSAA with IAccessible2 1.3 (`IA2 `_) + - ``ia2`` + - Windows + - Loading module `ia2_api_all.idl `_ with `comtypes `_ +``` The APIs are exposed through a pytest fixture with the name in the table above. The pytest fixture returns a wrapped version of the API. Requesting this