484 lines
16 KiB
Go
484 lines
16 KiB
Go
// Cthulhu screen reader.
|
|
// Copyright (c) 2026 Storm Dragon
|
|
// Authors: Storm Dragon, Angus Kola
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify it
|
|
// under the terms of the GNU General Public License as published by the Free
|
|
// Software Foundation, either version 3 of the License, or (at your option)
|
|
// any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful, but WITHOUT
|
|
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
// more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License along with
|
|
// this program. If not, see <https://www.gnu.org/licenses/gpl-3.0.html>.
|
|
|
|
package atspi
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestNativeBrowserLiveRegion(t *testing.T) {
|
|
if os.Getenv("CTHULHU_NATIVE_BROWSER") != "1" {
|
|
t.Skip("requires private browser session")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
|
defer cancel()
|
|
adapter := New()
|
|
if err := adapter.Start(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
stop, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
if err := adapter.Stop(stop); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
window, err := exec.CommandContext(ctx, "xdotool", "search", "--onlyvisible", "--name", "Cthulhu Browser Probe").Output()
|
|
if err != nil || len(strings.Fields(string(window))) == 0 {
|
|
t.Fatalf("browser window: %s: %v", window, err)
|
|
}
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "windowfocus", strings.Fields(string(window))[0]).CombinedOutput(); err != nil {
|
|
t.Fatalf("focus: %s: %v", out, err)
|
|
}
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "key", "F9").CombinedOutput(); err != nil {
|
|
t.Fatalf("trigger live region: %s: %v", out, err)
|
|
}
|
|
deadline := time.NewTimer(5 * time.Second)
|
|
defer deadline.Stop()
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type != ChildrenAddedEvent && event.Type != TextInsertedEvent {
|
|
continue
|
|
}
|
|
t.Logf("update type=%s role=%s live=%t politeness=%q text=%q payload=%q", event.Type, event.Source.Role, event.Source.LiveRegion, event.Source.LivePoliteness, event.Source.Text, event.Payload)
|
|
if event.Source.LiveRegion && event.Source.LivePoliteness == "polite" &&
|
|
(strings.Contains(event.Source.Text, "Username: Test message") || strings.Contains(event.Payload, "Username: Test message")) {
|
|
return
|
|
}
|
|
case <-deadline.C:
|
|
t.Fatal("active browser live-region update was not captured")
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeBrowser(t *testing.T) {
|
|
if os.Getenv("CTHULHU_NATIVE_BROWSER") != "1" {
|
|
t.Skip("requires private browser session")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
|
|
defer cancel()
|
|
adapter := New()
|
|
if err := adapter.Start(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
stop, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
if err := adapter.Stop(stop); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
// Session services can map windows after the launcher first focuses Brave.
|
|
window, err := exec.CommandContext(ctx, "xdotool", "search", "--onlyvisible", "--name", "Cthulhu Browser Probe").Output()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
windows := strings.Fields(string(window))
|
|
if len(windows) == 0 {
|
|
t.Fatal("browser window missing")
|
|
}
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "windowfocus", windows[0]).CombinedOutput(); err != nil {
|
|
t.Fatalf("focus: %s: %v", out, err)
|
|
}
|
|
tree, err := adapter.CaptureDesktop(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"Browser foundation", "Reading section", "Example link", "Display name", "Secret", "Remember choice", "Details"} {
|
|
ref := findNamedReference(t, tree, name)
|
|
snapshot, err := tree.Resolve(ref)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("%s: role=%s text=%q", name, snapshot.Role, snapshot.Text)
|
|
}
|
|
|
|
heading, err := tree.Resolve(findNamedReference(t, tree, "Reading section"))
|
|
if err != nil || heading.HeadingLevel != 2 {
|
|
t.Fatalf("heading level=%d err=%v", heading.HeadingLevel, err)
|
|
}
|
|
required, err := tree.Resolve(findNamedReference(t, tree, "Display name"))
|
|
if err != nil || !required.Required || !required.Editable {
|
|
t.Fatalf("entry semantics: %+v %v", required, err)
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
focusable, popup bool
|
|
}{
|
|
{"Reading section", true, false},
|
|
{"Ordinary unfocusable heading", false, false},
|
|
{"Display name", true, false},
|
|
{"Details", true, false},
|
|
{"Popup actions", true, true},
|
|
{"Popup toggle", true, true},
|
|
{"Ordinary action", true, false},
|
|
} {
|
|
snapshot, err := tree.Resolve(findNamedReference(t, tree, tc.name))
|
|
if err != nil || !snapshot.StatesKnown || snapshot.Focusable != tc.focusable || snapshot.HasPopup != tc.popup {
|
|
t.Fatalf("native mode states for %s: %+v, %v", tc.name, snapshot, err)
|
|
}
|
|
}
|
|
pageRef := findNamedReference(t, tree, "Browser foundation")
|
|
pageContext, err := tree.DocumentContext(pageRef)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
embedded := findNamedReference(t, tree, "Embedded heading")
|
|
embeddedContext, err := tree.DocumentContext(embedded)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ordinary := findNamedReference(t, tree, "Ordinary unfocusable heading")
|
|
ordinaryContext, err := tree.DocumentContext(ordinary)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if embeddedContext.TopLevel != pageContext.TopLevel || embeddedContext.Document == pageContext.Document ||
|
|
ordinaryContext.Document != pageContext.Document {
|
|
t.Fatalf("document ownership: embedded=%+v ordinary=%+v page=%+v", embeddedContext, ordinaryContext, pageContext)
|
|
}
|
|
chromeRef := findNamedReference(t, tree, "Address and search bar")
|
|
if _, err := tree.DocumentContext(chromeRef); err != ErrNotInDocument {
|
|
t.Fatalf("browser chrome belongs to document: %v", err)
|
|
}
|
|
// Browser chrome has a version-dependent number of Tab stops. Locate the
|
|
// first page heading with a bounded real-key traversal, then verify order.
|
|
expected := []string{"Browser foundation", "Reading section", "Example link", "Display name", "Secret", "Remember choice", "Details"}
|
|
next := 0
|
|
for attempts := 0; attempts < 35 && next < len(expected); attempts++ {
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "key", "Tab").CombinedOutput(); err != nil {
|
|
t.Fatalf("keys: %s: %v", out, err)
|
|
}
|
|
timer := time.NewTimer(500 * time.Millisecond)
|
|
events:
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type != "focus:" && event.Type != "object:state-changed:focused" {
|
|
continue
|
|
}
|
|
if event.Source.Name == expected[next] {
|
|
if !event.Source.StatesKnown || !event.Source.Focusable || !event.Source.Focused {
|
|
t.Fatalf("native focus states missing: %+v", event.Source)
|
|
}
|
|
if event.Source.Name == "Secret" && (event.Source.Text != "" || event.Source.CaretCharacter != "" || event.Source.TextBeforeCaret != "") {
|
|
t.Fatal("password snapshot contains text")
|
|
}
|
|
t.Logf("focus %s: %s", event.Source.Name, event.Source.Role)
|
|
next++
|
|
timer.Stop()
|
|
break events
|
|
}
|
|
if next > 0 {
|
|
t.Fatalf("focus %q; expected %q", event.Source.Name, expected[next])
|
|
}
|
|
case <-timer.C:
|
|
break events
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
if next != len(expected) {
|
|
t.Fatalf("only reached %d of %d page controls", next, len(expected))
|
|
}
|
|
waitEvent := func(name, kind string, state func(Snapshot) bool) {
|
|
t.Helper()
|
|
timeout := time.NewTimer(3 * time.Second)
|
|
defer timeout.Stop()
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Source.Name == name && event.Type == kind && state(event.Source) {
|
|
return
|
|
}
|
|
case <-timeout.C:
|
|
t.Fatalf("missing %s event for %s", kind, name)
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
key := func(keys ...string) {
|
|
t.Helper()
|
|
if out, err := exec.CommandContext(ctx, "xdotool", append([]string{"key"}, keys...)...).CombinedOutput(); err != nil {
|
|
t.Fatalf("keys: %s: %v", out, err)
|
|
}
|
|
}
|
|
key("space")
|
|
waitEvent("Details", "object:state-changed:expanded", func(s Snapshot) bool { return s.Expanded })
|
|
key("F8")
|
|
waitEvent("Cthulhu Browser Probe updated", AccessibleNameEvent, func(s Snapshot) bool {
|
|
if s.ToolkitName != "Chromium" {
|
|
t.Fatalf("browser name event toolkit = %q", s.ToolkitName)
|
|
}
|
|
return true
|
|
})
|
|
key("shift+Tab")
|
|
waitEvent("Remember choice", "object:state-changed:focused", func(s Snapshot) bool { return true })
|
|
key("space")
|
|
waitEvent("Remember choice", "object:state-changed:checked", func(s Snapshot) bool { return s.Checked })
|
|
// Move between page controls before checking focus loss. Chromium can keep
|
|
// the DOM's focused state when its address bar owns desktop focus.
|
|
tree, err = adapter.CaptureDesktop(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previous, err := tree.Resolve(findNamedReference(t, tree, "Details"))
|
|
if err != nil || !previous.StatesKnown || previous.Focused {
|
|
t.Fatalf("recapture retained old focused state: %+v, %v", previous, err)
|
|
}
|
|
key("ctrl+l")
|
|
// Recapture after scrolling/focus events before resolving chrome ownership.
|
|
waitEvent("Address and search bar", "object:state-changed:focused", func(s Snapshot) bool { return s.Editable })
|
|
tree, err = adapter.CaptureDesktop(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
chrome := findNamedReference(t, tree, "Address and search bar")
|
|
if _, err := tree.DocumentContext(chrome); err != ErrNotInDocument {
|
|
t.Fatalf("address bar document: %v", err)
|
|
}
|
|
foundInsertion, foundPopup := false, false
|
|
key("c")
|
|
timerTyping := time.NewTimer(250 * time.Millisecond)
|
|
readTyping:
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type == TextInsertedEvent && event.Payload == "c" && event.Source.Editable && event.Source.Autocomplete {
|
|
foundInsertion = true
|
|
}
|
|
if event.Source.Role == "list item" && event.Source.AutocompletePopup && event.PopupFor == chrome.ID() {
|
|
foundPopup = true
|
|
}
|
|
case <-timerTyping.C:
|
|
break readTyping
|
|
}
|
|
}
|
|
if !foundInsertion || !foundPopup {
|
|
t.Fatalf("address semantics: insertion=%v popup=%v", foundInsertion, foundPopup)
|
|
}
|
|
key("Escape", "Escape", "Escape")
|
|
waitEvent("Remember choice", "object:state-changed:focused", func(s Snapshot) bool { return s.Checked })
|
|
pageRef = findNamedReference(t, tree, "Browser foundation")
|
|
key("ctrl+r")
|
|
loadTimer := time.NewTimer(5 * time.Second)
|
|
defer loadTimer.Stop()
|
|
waitLoad:
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type == DocumentLoadCompleteEvent {
|
|
if event.Source.Name != "Cthulhu Browser Probe" || event.Source.Text != "" {
|
|
t.Fatalf("loaded title snapshot: %+v", event.Source)
|
|
}
|
|
break waitLoad
|
|
}
|
|
case <-loadTimer.C:
|
|
t.Fatal("active document load announcement missing")
|
|
}
|
|
}
|
|
deadline := time.NewTimer(5 * time.Second)
|
|
defer deadline.Stop()
|
|
ticker := time.NewTicker(20 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
waitReload:
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
if _, err := tree.Resolve(pageRef); err != nil {
|
|
break waitReload
|
|
}
|
|
case <-deadline.C:
|
|
t.Fatal("browser reload retained old document reference")
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
// A fresh document must be discoverable after the adapter has already
|
|
// cached the browser tree, not only when connecting to a preloaded page.
|
|
for attempts := 0; attempts < 25; attempts++ {
|
|
key("Tab")
|
|
timer := time.NewTimer(250 * time.Millisecond)
|
|
waitFocus:
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type == FocusedStateEvent && event.Source.Name == "Browser foundation" {
|
|
timer.Stop()
|
|
captureCtx, stopCapture := context.WithTimeout(ctx, 2*time.Second)
|
|
captured, err := adapter.CaptureDocument(captureCtx, event.Source.ID())
|
|
stopCapture()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ref, err := captured.Reference(event.Source.ID())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
doc, err := captured.Document(ctx, ref)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
foundFragment := false
|
|
for _, node := range doc.Nodes {
|
|
if node.Snapshot.ElementID == "target" {
|
|
foundFragment = true
|
|
break
|
|
}
|
|
}
|
|
if !foundFragment {
|
|
t.Fatal("document capture omitted the target element ID")
|
|
}
|
|
root, err := captured.Root()
|
|
if err != nil || root != doc.Context.TopLevel || captured == tree {
|
|
t.Fatalf("document capture must own only the document: root=%v error=%v", root, err)
|
|
}
|
|
link := findNamedReference(t, captured, "Example link")
|
|
if err := adapter.ActivateDocumentObject(ctx, link, event.Source.ID()); err != nil {
|
|
t.Fatalf("activate captured link: %v", err)
|
|
}
|
|
activated := false
|
|
for attempts := 0; attempts < 20 && !activated; attempts++ {
|
|
updated, err := adapter.CaptureDocument(ctx, link.ID())
|
|
if err == nil {
|
|
root, rootErr := updated.Root()
|
|
if rootErr == nil {
|
|
document, documentErr := updated.Document(ctx, root)
|
|
if documentErr == nil {
|
|
for _, node := range document.Nodes {
|
|
if node.Snapshot.Name == "Text refreshed after the ordinary heading." {
|
|
activated = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if !activated {
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
}
|
|
if !activated {
|
|
t.Fatal("captured link action did not update the page")
|
|
}
|
|
testNativeBrowserBackgroundLoad(t, adapter)
|
|
return
|
|
}
|
|
case <-timer.C:
|
|
break waitFocus
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
t.Fatal("reloaded heading focus missing")
|
|
}
|
|
|
|
func testNativeBrowserBackgroundLoad(t *testing.T, adapter *Adapter) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
|
|
defer cancel()
|
|
requested, release := make(chan struct{}, 1), make(chan struct{})
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/hold" {
|
|
select {
|
|
case requested <- struct{}{}:
|
|
default:
|
|
}
|
|
select {
|
|
case <-release:
|
|
case <-r.Context().Done():
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/html")
|
|
_, _ = w.Write([]byte(`<!doctype html><title>Pending page</title><p>Content</p><img src="/hold" alt="Pending image">`))
|
|
}))
|
|
defer func() {
|
|
select {
|
|
case <-release:
|
|
default:
|
|
close(release)
|
|
}
|
|
server.Close()
|
|
}()
|
|
key := func(chord string) {
|
|
t.Helper()
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "key", "--clearmodifiers", chord).CombinedOutput(); err != nil {
|
|
t.Fatalf("keys: %s: %v", out, err)
|
|
}
|
|
}
|
|
key("ctrl+l")
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "type", "--clearmodifiers", server.URL).CombinedOutput(); err != nil {
|
|
t.Fatalf("type local URL: %s: %v", out, err)
|
|
}
|
|
key("Return")
|
|
select {
|
|
case <-requested:
|
|
case <-ctx.Done():
|
|
t.Fatal("page did not request its held resource")
|
|
}
|
|
// Discard focus events from navigating to the held page. Only a fresh
|
|
// address focus after Ctrl+L can release the response and complete loading.
|
|
for len(adapter.events) > 0 {
|
|
<-adapter.events
|
|
}
|
|
key("ctrl+l")
|
|
var chrome ObjectID
|
|
for !chrome.Valid() {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type == FocusedStateEvent && event.Source.Autocomplete && event.Source.Focused {
|
|
chrome = event.Source.ID()
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("address focus missing during page load")
|
|
}
|
|
}
|
|
close(release)
|
|
for {
|
|
select {
|
|
case event := <-adapter.Events():
|
|
if event.Type == DocumentLoadCompleteEvent && event.Source.Name == "Pending page" {
|
|
if event.FocusOutsideDocument != chrome {
|
|
t.Fatalf("load lost native chrome focus: got %v want %v", event.FocusOutsideDocument, chrome)
|
|
}
|
|
return
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("background load event missing")
|
|
}
|
|
}
|
|
}
|