Share native current-line speech with Remote

This commit is contained in:
ak47 committed 2026-11-05 17:55:11 -05:00
1 parent 103fbfd81a
commit ee3d91af23
11 files changed
+289 -16

No files matched your search

+9 -5
View File
@@ -15,7 +15,7 @@ func (handler *eventReviewHandler) current_line(command input.Command) {
return
}
handler.syncForeground()
if handler.objectReview == nil || !handler.state.current.ID().Valid() {
if handler.focusGate.Locked() || handler.focusGate.Suspended() || handler.objectReview == nil || !handler.state.current.ID().Valid() {
return
}
handler.state.pending.terminalFocus.clear()
@@ -34,7 +34,7 @@ func (handler *eventReviewHandler) current_line(command input.Command) {
}
func (handler *eventReviewHandler) present_current_line(result review.Result) {
if !handler.objectReview.CurrentDocumentResult(result) || result.DocumentFocus != handler.objectReview.Focus() || handler.focusGate.Suspended() {
if !handler.objectReview.CurrentDocumentResult(result) || result.DocumentFocus != handler.objectReview.Focus() || handler.focusGate.Locked() || handler.focusGate.Suspended() {
return
}
if result.CurrentLineBrowse && (handler.browser == nil || !handler.browser.acceptsDocumentResult(review.Result{
@@ -44,9 +44,13 @@ func (handler *eventReviewHandler) present_current_line(result review.Result) {
}
request := speechRequest{Text: "Could not read current line.", Role: "system", Cause: "current-line"}
if result.Err == nil {
// Native focused-line reads do not carry complete ancestry. Only
// browse reports with captured source metadata opt into observation.
if result.CurrentLineBrowse {
// Native reports require the matching focused source and an unbroken
// capture through the application. Missing privacy data stays local.
context := result.DocumentCaretContext
nativeSource := len(context) > 0 && result.DocumentFocus.Valid() &&
result.DocumentPosition.Snapshot.ID() == result.DocumentFocus && context[0].ID() == result.DocumentFocus &&
context[len(context)-1].Role == "application"
if result.CurrentLineBrowse || nativeSource {
sources := atspi.DocumentLine{Contents: []atspi.DocumentContent{{Snapshot: result.DocumentPosition.Snapshot}},
SourceContexts: [][]atspi.Snapshot{result.DocumentCaretContext}}
request.origin = document_line_speech_origin(sources)
+35
View File
@@ -98,3 +98,38 @@ func TestCurrentLinePresentationAndStaleResults(t *testing.T) {
t.Fatal("report from previous focus was spoken")
}
}
func TestCurrentLineSecureFocus(t *testing.T) {
for _, mode := range []string{"lock", "handoff", "foreground-lock", "foreground-handoff"} {
t.Run(mode, func(t *testing.T) {
focus := atspi.NewObjectID(":1.42", "/focus")
worker, _ := review.NewWorker(&fakeAdapter{}, time.Second)
worker.SyncFocus(focus)
queue := &fakeSpeechQueue{requests: make(chan speechRequest, 1), spoken: make(chan string, 1)}
handler := eventReviewHandler{state: &eventLoopState{}, objectReview: worker, speech: queue,
logger: slog.New(slog.DiscardHandler), focusGate: &focusGate{}}
handler.state.current = atspi.Snapshot{BusName: focus.BusName, Path: focus.Path, Role: "text"}
secure := func() {
if mode == "lock" || mode == "foreground-lock" {
handler.focusGate.SetLocked(true, nil, nil, nil, nil, &brailleWorker{})
} else {
handler.focusGate.SetSuspended(true)
}
}
if mode == "lock" || mode == "handoff" {
secure()
handler.syncForeground = func() { t.Fatal("secure command synchronized foreground") }
} else {
handler.syncForeground = secure
}
handler.current_line(input.PresentCurrentLine)
// An already completed read must also be rejected on secure input.
result := native_current_line_result()
result.DocumentFocus = focus
handler.present_current_line(result)
if len(queue.requests) != 0 {
t.Fatal("current line spoke during lock or handoff")
}
})
}
}
@@ -69,6 +69,7 @@ func TestRemoteCurrentLineSourcePrivacy(t *testing.T) {
SourceContexts: [][]atspi.Snapshot{ancestors, neighbor}}
case "native":
result.CurrentLineBrowse = false
result.DocumentCaretContext = nil
case "failure":
result.Err = errors.New("fixture unavailable")
}
@@ -132,3 +133,91 @@ func TestRemoteCurrentLineResolvedOutput(t *testing.T) {
}
}
}
func native_current_line_result() review.Result {
source := atspi.Snapshot{BusName: ":1.42", Path: "/focus", Role: "text"}
return review.Result{DocumentCaretText: "Native colour.", DocumentPosition: review.DocumentPosition{Snapshot: source},
DocumentCaretContext: []atspi.Snapshot{source, {BusName: source.BusName, Path: "/window", Role: "frame"},
{BusName: source.BusName, Path: "/app", Role: "application"}}}
}
func TestRemoteNativeCurrentLineSourcePrivacy(t *testing.T) {
for _, privacy := range []string{"ordinary", "password", "ancestor", "unknown", "invalid", "defunct", "missing", "truncated", "mismatched", "foreign", "failure"} {
t.Run(privacy, func(t *testing.T) {
result := native_current_line_result()
switch privacy {
case "password":
result.DocumentPosition.Snapshot.Role = "password text"
case "ancestor":
result.DocumentCaretContext[1].Role = "password text"
case "unknown":
result.DocumentCaretContext[1].Role = "unknown"
case "invalid":
result.DocumentCaretContext[1].Role = "invalid"
case "defunct":
result.DocumentCaretContext[1].Defunct = true
case "missing":
result.DocumentCaretContext = nil
case "truncated":
result.DocumentCaretContext = result.DocumentCaretContext[:2]
case "mismatched":
result.DocumentCaretContext[0].Path = "/other"
case "foreign":
result.DocumentPosition.Snapshot.Path = "/other"
result.DocumentCaretContext[0].Path = "/other"
case "failure":
result.Err = errors.New("fixture unavailable")
}
request := current_line_observation_request(t, result)
if exportable_speech(request) != (privacy == "ordinary") {
t.Fatalf("native source %s produced exportable=%v", privacy, exportable_speech(request))
}
if privacy != "failure" && request.Text != result.DocumentCaretText {
t.Fatal("privacy classification changed local native reading")
}
})
}
}
func TestRemoteNativeCurrentLineResolvedOutput(t *testing.T) {
worker, session, _, _ := observation_fixture(t)
worker.textTransform = func(text string) string { return strings.ReplaceAll(text, "colour", "color") }
for _, probe := range []struct {
style review.FlatSpeech
text string
want []string
characters bool
}{
{review.FlatSpeak, "Native colour.", []string{"Native color."}, false},
{review.FlatSpell, "book", []string{"b", "o", "o", "k"}, true},
{review.FlatPhonetic, "book", []string{"bravo", "oscar", "oscar", "kilo"}, false},
{review.FlatSpeak, "\n", []string{"blank"}, false},
} {
result := native_current_line_result()
result.FlatSpeech, result.DocumentCaretText = probe.style, probe.text
for _, private := range []bool{false, true} {
if private {
result.DocumentCaretContext[1].Role = "password text"
}
if err := worker.EnqueueAndWait(t.Context(), current_line_observation_request(t, result)); err != nil {
t.Fatal(err)
}
for _, want := range probe.want {
if !private {
value := observed_speech(t, session)
if value.Kind != "speak" || len(value.Parts) != 2 || value.Parts[0].Text != want ||
value.Parts[0].Characters != probe.characters || !value.Parts[1].End {
t.Fatal("native report lost resolved output or spelling boundaries", value)
}
}
if got := <-worker.speaker.(*observationSpeaker).spoken; got != want {
t.Fatalf("native local reading %q, want %q", got, want)
}
}
worker.InterruptForUser()
if value := observed_speech(t, session); value.Kind != "cancel" {
t.Fatal("private native output reached the child", value)
}
}
}
}
@@ -572,6 +572,16 @@ func TestRemoteConnectionCommandPrivateRelay(t *testing.T) {
t.Fatal(err)
}
}
for _, private := range []bool{false, true} {
result := native_current_line_result()
if private {
result.DocumentCaretText = "private native text"
result.DocumentCaretContext[1].Role = "password text"
}
if err := worker.EnqueueAndWait(t.Context(), current_line_observation_request(t, result)); err != nil {
t.Fatal(err)
}
}
for _, kind := range []string{"object", "line"} {
for _, privacy := range []string{"ordinary", "snapshot-ancestor", "stale"} {
request := review_observation_request(t, kind, review_observation_result(t, kind, privacy))
@@ -620,7 +630,7 @@ func TestRemoteConnectionCommandPrivateRelay(t *testing.T) {
t.Fatal("reader output did not reach the relay in order")
}
}
if received.String() != "color "+remoteText+"First color.Second color.Current color.Review color.staticFlat color. Review color.Review color.static" || frames < 10 || ends != 9 {
if received.String() != "color "+remoteText+"First color.Second color.Current color.Native color.Review color.staticFlat color. Review color.Review color.static" || frames < 11 || ends != 10 {
t.Fatalf("observation leaked private text or lost resolved output/boundaries: frames=%d ends=%d suffix=%q", frames, ends, received.String()[max(0, received.Len()-160):])
}
}
+63 -2
View File
@@ -13,6 +13,7 @@ import (
"context"
"errors"
"strings"
"time"
)
// ReadFocusedLine reads only the line containing the current native caret.
@@ -21,7 +22,7 @@ func (adapter *Adapter) ReadFocusedLine(ctx context.Context, focus ObjectID) (Do
return result.content, err
}
func (adapter *Adapter) read_native_focused_line(ctx context.Context, focus ObjectID) (DocumentContent, error) {
func (adapter *Adapter) read_native_focused_line(ctx context.Context, focus ObjectID) (content DocumentContent, err error) {
if err := ctx.Err(); err != nil {
return DocumentContent{}, err
}
@@ -29,11 +30,17 @@ func (adapter *Adapter) read_native_focused_line(ctx context.Context, focus Obje
return DocumentContent{}, ErrStaleObject
}
source := adapter.nativeFocus[len(adapter.nativeFocus)-1].source
C.atspi_accessible_clear_cache_single(source)
snapshot := snapshotFromNativeWithCaret(source, false, false, false, false)
if snapshot.Role == "password text" {
return DocumentContent{}, ErrNoDocumentCaret
}
content := DocumentContent{Snapshot: snapshot, Text: snapshot.Name}
content = DocumentContent{Snapshot: snapshot, Text: snapshot.Name}
defer func() {
if err == nil {
content.SourceContext = native_focused_line_context(ctx, source)
}
}()
text := C.atspi_accessible_get_text_iface(source)
if text == nil {
return content, ctx.Err()
@@ -84,3 +91,57 @@ func (adapter *Adapter) read_native_focused_line(ctx context.Context, focus Obje
content.Start, content.End, content.Text, content.RawText = line.Start, line.End, line.Text, line.Text
return content, ctx.Err()
}
// Privacy capture is separate from presentation context: include every parent,
// even unnamed containers and parents outside the window. Keep it on the GLib
// owner and bound the walk without scanning children or retaining native refs.
// Failed/truncated capture must not turn a successful local read into an error.
func native_focused_line_context(ctx context.Context, source *C.AtspiAccessible) []Snapshot {
deadline, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
defer cancel()
current := (*C.AtspiAccessible)(C.g_object_ref(C.gpointer(source)))
defer func() {
if current != nil {
C.g_object_unref(C.gpointer(current))
}
}()
var result []Snapshot
seen := make(map[ObjectID]bool)
for current != nil && len(result) < 64 && deadline.Err() == nil {
C.atspi_accessible_clear_cache_single(current)
id := nativeObjectID(current)
if !id.Valid() || seen[id] {
return nil
}
seen[id] = true
var nativeErr *C.GError
roleName := C.atspi_accessible_get_role_name(current, &nativeErr)
role := C.GoString(roleName)
C.g_free(C.gpointer(roleName))
if nativeErr != nil {
C.g_error_free(nativeErr)
return nil
}
states := C.atspi_accessible_get_state_set(current)
if states == nil {
return nil
}
defunct := C.atspi_state_set_contains(states, C.ATSPI_STATE_DEFUNCT) != 0
C.g_object_unref(C.gpointer(states))
result = append(result, Snapshot{BusName: id.BusName, Path: id.Path, Role: role, Defunct: defunct})
if role == "application" {
if deadline.Err() != nil {
return nil
}
return result
}
parent := C.atspi_accessible_get_parent(current, &nativeErr)
C.g_object_unref(C.gpointer(current))
current = parent
if nativeErr != nil {
C.g_error_free(nativeErr)
return nil
}
}
return nil
}
+29 -1
View File
@@ -55,6 +55,7 @@ func TestNativeCurrentLine(t *testing.T) {
{"text", "Final\n", 6, "", false},
{"password", "private test text", 5, "", true},
{"button", "", 0, "Example button", false},
{"deep", "Deep local line", 3, "Deep local line", false},
} {
command, _ := json.Marshal(map[string]any{"action": "focus", "control": probe.control, "text": probe.text, "offset": probe.offset})
fmt.Fprintln(stdin, string(command))
@@ -64,7 +65,7 @@ func TestNativeCurrentLine(t *testing.T) {
select {
case event := <-adapter.Events():
if event.Type == "object:state-changed:focused" && event.Detail1 == 1 &&
(event.Source.Name == map[string]string{"entry": "Entry", "text": "Text", "password": "Password", "button": "Example button"}[probe.control]) {
(event.Source.Name == map[string]string{"entry": "Entry", "text": "Text", "password": "Password", "button": "Example button", "deep": "Deep entry"}[probe.control]) {
focus = event.Source.ID()
goto focused
}
@@ -85,6 +86,33 @@ func TestNativeCurrentLine(t *testing.T) {
if err != nil || strings.TrimRight(content.Text, "\r\n") != probe.want {
t.Fatalf("%s offset %d: %+v %v, want %q", probe.control, probe.offset, content, err, probe.want)
}
if probe.control == "deep" {
if len(content.SourceContext) != 0 {
t.Fatal("truncated native ancestry authorized export")
}
} else if len(content.SourceContext) == 0 || content.SourceContext[0].ID() != focus ||
content.SourceContext[len(content.SourceContext)-1].Role != "application" {
t.Fatalf("native report lost complete source ancestry: %+v", content.SourceContext)
}
if probe.control == "entry" && probe.want != "" {
for _, role := range []struct{ atk, want string }{{"PASSWORD_TEXT", "password text"}, {"UNKNOWN", "unknown"}, {"INVALID", "invalid"}} {
fmt.Fprintf(stdin, "{\"action\":\"privacy\",\"role\":%q}\n", role.atk)
ready()
private, err := adapter.ReadFocusedLine(ctx, focus)
if err != nil || private.Text != content.Text {
t.Fatal("ancestor privacy changed local text", err)
}
found := false
for _, ancestor := range private.SourceContext {
found = found || ancestor.Role == role.want
}
if len(private.SourceContext) != 0 && !found {
t.Fatal("native capture omitted a private ancestor", role.want)
}
}
fmt.Fprintln(stdin, "{\"action\":\"privacy\",\"role\":\"FILLER\"}")
ready()
}
if probe.control != "button" {
fmt.Fprintf(stdin, "{\"action\":\"caret\",\"control\":%q}\n", probe.control)
if !scanner.Scan() || scanner.Text() != fmt.Sprint(probe.offset) {
+3
View File
@@ -15,6 +15,9 @@ type DocumentContent struct {
End int
Text string
RawText string
// SourceContext is the focused-line source followed by its ancestors through
// the application. Nil means privacy capture failed; local text stays usable.
SourceContext []Snapshot
// Separator is display spacing before this object, outside its native range.
Separator string
Bounds Rectangle
+1
View File
@@ -44,6 +44,7 @@ func (worker *Worker) read_current_line(ctx context.Context, cursor *DocumentCur
result.Err = err
result.DocumentCaretText, result.DocumentCaretRawText = content.Text, content.RawText
result.DocumentPosition.Snapshot = content.Snapshot
result.DocumentCaretContext = content.SourceContext
return result
}
// Start a private cursor at the saved browse position. Going to line start
+16 -1
View File
@@ -22,7 +22,9 @@ func (capturer *currentLineCapturer) ReadFocusedLine(context.Context, atspi.Obje
}
func TestCurrentLineUsesNativeFocusOutsideBrowse(t *testing.T) {
capturer := &currentLineCapturer{content: atspi.DocumentContent{Text: "Native caret line"}}
source := atspi.Snapshot{BusName: webID("/entry").BusName, Path: "/entry", Role: "text"}
capturer := &currentLineCapturer{content: atspi.DocumentContent{Text: "Native caret line", Snapshot: source,
SourceContext: []atspi.Snapshot{source, {Role: "frame"}, {Role: "application"}}}}
worker, cancel := startWorker(t, capturer, time.Second)
defer cancel()
worker.SyncFocus(webID("/entry"))
@@ -34,12 +36,25 @@ func TestCurrentLineUsesNativeFocusOutsideBrowse(t *testing.T) {
if !result.CurrentLine || result.CurrentLineBrowse || result.FlatSpeech != style || result.Err != nil || result.DocumentCaretText != "Native caret line" {
t.Fatalf("native current line: %+v", result)
}
if !reflect.DeepEqual(result.DocumentCaretContext, capturer.content.SourceContext) {
t.Fatal("native current line lost its owned privacy context")
}
}
if capturer.reads != 3 {
t.Fatal("native line was cached")
}
}
func TestCurrentLineNativeMissingContextPreservesLocalText(t *testing.T) {
capturer := &currentLineCapturer{content: atspi.DocumentContent{Text: "Local line"}}
worker, _ := NewWorker(capturer, time.Second)
worker.SyncFocus(webID("/entry"))
result := worker.read_current_line(t.Context(), nil, false)
if result.Err != nil || result.DocumentCaretText != "Local line" || len(result.DocumentCaretContext) != 0 {
t.Fatalf("missing privacy metadata changed local reading: %+v", result)
}
}
func TestCurrentLineReadsWholeBrowseLineWithoutMoving(t *testing.T) {
reader := caretTextReader{webID("/text"): "First line\nSecond café 猫\nThird", webID("/next"): "Next"}
cursor := newCaretTextCursor(t, reader)