Files
cthulhu/internal/app/document_review.go
ak47 49be8309fe Enable isolated Remote speech and protect reading exports
Attach incoming software speech to reader sessions with lease checks, local output priority and isolated cancellation. Preserve every document fragment source for privacy checks and export reviewed browse current-line reading, spelling and phonetic output.

Verified focused reading and Remote race tests, complete review and external Remote race suites, private relay integration, speech regressions, affected-package vet, reader build and Tentacles make check. Audible speech, physical input/Braille and live interoperability remain unverified.
2026-11-05 15:41:08 -05:00

477 lines
19 KiB
Go

// Cthulhu screen reader.
// Copyright (c) 2026 Storm Dragon
// Authors: Storm Dragon, Angus Kola
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along with
// this program. If not, see <https://www.gnu.org/licenses/gpl-3.0.html>.
package app
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"git.stormux.org/storm/cthulhu/internal/atspi"
"git.stormux.org/storm/cthulhu/internal/input"
"git.stormux.org/storm/cthulhu/internal/presentation"
"git.stormux.org/storm/cthulhu/internal/review"
"git.stormux.org/storm/cthulhu/internal/sound"
)
func documentReviewCommand(command input.Command) (review.DocumentTarget, int, bool) {
switch command {
case input.DocumentNextHeading:
return review.DocumentHeading, 0, false
case input.DocumentPreviousHeading:
return review.DocumentHeading, 0, true
case input.DocumentNextHeadingAtLevel1:
return review.DocumentHeading, 1, false
case input.DocumentPreviousHeadingAtLevel1:
return review.DocumentHeading, 1, true
case input.DocumentNextHeadingAtLevel2:
return review.DocumentHeading, 2, false
case input.DocumentPreviousHeadingAtLevel2:
return review.DocumentHeading, 2, true
case input.DocumentNextHeadingAtLevel3:
return review.DocumentHeading, 3, false
case input.DocumentPreviousHeadingAtLevel3:
return review.DocumentHeading, 3, true
case input.DocumentNextHeadingAtLevel4:
return review.DocumentHeading, 4, false
case input.DocumentPreviousHeadingAtLevel4:
return review.DocumentHeading, 4, true
case input.DocumentNextHeadingAtLevel5:
return review.DocumentHeading, 5, false
case input.DocumentPreviousHeadingAtLevel5:
return review.DocumentHeading, 5, true
case input.DocumentNextHeadingAtLevel6:
return review.DocumentHeading, 6, false
case input.DocumentPreviousHeadingAtLevel6:
return review.DocumentHeading, 6, true
case input.DocumentNextLink:
return review.DocumentLink, 0, false
case input.DocumentPreviousLink:
return review.DocumentLink, 0, true
case input.DocumentNextButton:
return review.DocumentButton, 0, false
case input.DocumentPreviousButton:
return review.DocumentButton, 0, true
case input.DocumentNextCheckBox:
return review.DocumentCheckBox, 0, false
case input.DocumentPreviousCheckBox:
return review.DocumentCheckBox, 0, true
case input.DocumentNextList:
return review.DocumentList, 0, false
case input.DocumentPreviousList:
return review.DocumentList, 0, true
case input.DocumentNextSeparator:
return review.DocumentSeparator, 0, false
case input.DocumentPreviousSeparator:
return review.DocumentSeparator, 0, true
case input.DocumentNextBlockQuote:
return review.DocumentBlockQuote, 0, false
case input.DocumentPreviousBlockQuote:
return review.DocumentBlockQuote, 0, true
case input.DocumentNextComboBox:
return review.DocumentComboBox, 0, false
case input.DocumentPreviousComboBox:
return review.DocumentComboBox, 0, true
case input.DocumentNextEntry:
return review.DocumentEntry, 0, false
case input.DocumentPreviousEntry:
return review.DocumentEntry, 0, true
case input.DocumentNextFormField:
return review.DocumentFormField, 0, false
case input.DocumentPreviousFormField:
return review.DocumentFormField, 0, true
case input.DocumentNextImage:
return review.DocumentImage, 0, false
case input.DocumentPreviousImage:
return review.DocumentImage, 0, true
case input.DocumentNextLandmark:
return review.DocumentLandmark, 0, false
case input.DocumentPreviousLandmark:
return review.DocumentLandmark, 0, true
case input.DocumentNextListItem:
return review.DocumentListItem, 0, false
case input.DocumentPreviousListItem:
return review.DocumentListItem, 0, true
case input.DocumentNextLiveRegion:
return review.DocumentLiveRegion, 0, false
case input.DocumentPreviousLiveRegion:
return review.DocumentLiveRegion, 0, true
case input.DocumentNextParagraph:
return review.DocumentParagraph, 0, false
case input.DocumentPreviousParagraph:
return review.DocumentParagraph, 0, true
case input.DocumentNextRadioButton:
return review.DocumentRadioButton, 0, false
case input.DocumentPreviousRadioButton:
return review.DocumentRadioButton, 0, true
case input.DocumentNextTable:
return review.DocumentTable, 0, false
case input.DocumentPreviousTable:
return review.DocumentTable, 0, true
case input.DocumentNextUnvisitedLink:
return review.DocumentUnvisitedLink, 0, false
case input.DocumentPreviousUnvisitedLink:
return review.DocumentUnvisitedLink, 0, true
case input.DocumentNextVisitedLink:
return review.DocumentVisitedLink, 0, false
case input.DocumentPreviousVisitedLink:
return review.DocumentVisitedLink, 0, true
case input.DocumentNextClickable:
return review.DocumentClickable, 0, false
case input.DocumentPreviousClickable:
return review.DocumentClickable, 0, true
case input.DocumentNextChunk:
return review.DocumentChunk, 0, false
case input.DocumentPreviousChunk:
return review.DocumentChunk, 0, true
}
return 0, 0, false
}
func documentCaretCommand(command input.Command) review.DocumentCaretCommand {
switch command {
case input.DocumentTableLeft:
return review.TableLeft
case input.DocumentTableRight:
return review.TableRight
case input.DocumentTableUp:
return review.TableUp
case input.DocumentTableDown:
return review.TableDown
case input.DocumentTableFirst:
return review.TableFirst
case input.DocumentTableLast:
return review.TableLast
case input.DocumentCaretNextCharacter:
return review.CaretNextCharacter
case input.DocumentCaretPreviousCharacter:
return review.CaretPreviousCharacter
case input.DocumentCaretNextWord:
return review.CaretNextWord
case input.DocumentCaretPreviousWord:
return review.CaretPreviousWord
case input.DocumentCaretNextLine:
return review.CaretNextLine
case input.DocumentCaretPreviousLine:
return review.CaretPreviousLine
case input.DocumentCaretLineStart:
return review.CaretLineStart
case input.DocumentCaretLineEnd:
return review.CaretLineEnd
case input.DocumentCaretDocumentStart:
return review.CaretDocumentStart
case input.DocumentCaretDocumentEnd:
return review.CaretDocumentEnd
}
return 0
}
func presentDocumentCaretResult(logger *slog.Logger, speech speechEnqueuer, result review.Result, options presentation.Options) {
if logger.Enabled(context.Background(), slog.LevelDebug) {
logger.Debug("document caret navigation completed", "elapsed", result.Elapsed,
"error", result.Err, "document_epoch", result.DocumentEpoch,
"focus", result.DocumentFocus, "document", result.DocumentPosition.Document.ID(),
"application", result.DocumentPosition.Snapshot.ApplicationName, "role", result.DocumentPosition.Snapshot.Role,
"navigation", result.DocumentDiagnostics,
"capture_duration", result.CaptureDuration, "capture_reason", result.CaptureReason,
"moved", result.Moved, "command", result.DocumentCaretCommand.String(),
"bus", result.DocumentPosition.Object.ID().BusName,
"source_path", result.DocumentPosition.Object.ID().Path,
"virtual_caret_offset", result.DocumentPosition.Offset,
"native_text_start", result.DocumentCaretStart, "native_text_end", result.DocumentCaretEnd,
"line_fragments", document_line_ranges(result.DocumentCaretLine),
"caret_sync_error", result.CaretErr,
"braille_line_available", result.DocumentBrailleLine != nil || result.DocumentCaretLine != nil,
"braille_read_error", result.DocumentBrailleErr)
}
if result.Err != nil {
logger.Warn("document caret navigation failed", "error", result.Err)
speech.Enqueue(speechRequest{Text: "Could not find current location.", Role: "system", Cause: "document-caret"})
return
}
if !result.Moved && result.DocumentCaretText == "" {
return
}
text := result.DocumentCaretText
if text == " " {
text = "space"
}
if text == "\n" {
if !options.SpeakBlankLines {
return
}
text = "blank"
}
request := speechRequest{origin: document_speech_origin(result.DocumentPosition.Snapshot, result.DocumentCaretContext), Text: text, Role: result.DocumentPosition.Snapshot.Role, Cause: "document-caret"}
if result.DocumentCaretLine != nil {
if origin := document_line_speech_origin(*result.DocumentCaretLine); origin != speechOriginLocal {
request.origin = origin
}
}
request.PhoneticHint = result.DocumentCaretCommand == review.CaretNextCharacter || result.DocumentCaretCommand == review.CaretPreviousCharacter
request.Context = result.DocumentCaretContext
if len(request.Context) > 0 && request.Context[0].ID() == result.DocumentPosition.Snapshot.ID() {
request.Context = request.Context[1:]
}
if result.DocumentCaretCommand >= review.TableLeft && result.DocumentCaretCommand <= review.TableLast {
snapshot := result.DocumentPosition.Snapshot
prefix := result.DocumentTablePrefix
if result.Moved && options.SpeechVerbosityLevel != 0 && prefix != "" &&
(result.DocumentTableRowHeader != "" || result.DocumentTableColumnHeader != "") {
parts := []string{
presentation.TableHeaderText(result.DocumentTableRowHeader, "row header", options.SpeechVerbosityLevel),
presentation.TableHeaderText(result.DocumentTableColumnHeader, "column header", options.SpeechVerbosityLevel),
}
prefix = strings.TrimSpace(strings.Join(parts, " "))
if strings.HasPrefix(request.Text, result.DocumentTablePrefix+" ") {
request.Text = prefix + strings.TrimPrefix(request.Text, result.DocumentTablePrefix)
}
}
if result.Moved && options.Reading != nil && options.Reading.OnlySpeakDisplayedText {
request.Text = snapshot.Name
}
vertical := result.DocumentCaretCommand != review.TableLeft && result.DocumentCaretCommand != review.TableRight
if result.Moved && vertical && snapshot.TableRowText != "" && presentation.FullRowEnabled(snapshot, options) {
if options.Reading != nil && options.Reading.OnlySpeakDisplayedText {
request.Text = snapshot.TableRowText
} else {
request.Text = strings.TrimSpace(strings.Join([]string{prefix, snapshot.TableRowText, result.DocumentTableSuffix}, " "))
}
}
}
for _, ancestor := range result.DocumentCaretContext {
if ancestor.Role == "link" {
request.VoiceType = "hyperlink"
break
}
}
switch result.DocumentCaretCommand {
case review.CaretNextCharacter, review.CaretPreviousCharacter, review.CaretLineStart, review.CaretLineEnd:
request.Character = true
case review.CaretNextLine, review.CaretPreviousLine, review.CaretDocumentStart, review.CaretDocumentEnd,
review.CaretNextWord, review.CaretPreviousWord:
line := atspi.DocumentLine{Contents: []atspi.DocumentContent{{Snapshot: result.DocumentPosition.Snapshot, Text: text, RawText: result.DocumentCaretRawText}}}
if result.DocumentCaretCommand != review.CaretNextWord && result.DocumentCaretCommand != review.CaretPreviousWord {
request.Line = document_speech_line(line)
}
roles := map[string]bool{result.DocumentPosition.Snapshot.Role: true}
// Chromium commonly exposes link/heading text in static descendants.
// Python's web _generateRoleName includes those ancestor roles. Keep
// their identity separate from the leaf which owns the caret offset.
for _, ancestor := range result.DocumentCaretContext {
if (ancestor.Role != "link" && ancestor.Role != "heading") || roles[ancestor.Role] {
continue
}
roles[ancestor.Role] = true
ancestor.Name = ""
line.Contents = append(line.Contents, atspi.DocumentContent{Snapshot: ancestor})
}
spoken := presentation.DocumentLineSpeechWithOptions(line, options)
request.Text, request.Chunks = spoken.Text, spoken.Chunks
request.Sound = presentation.DocumentLineSoundCues(line, options)
fallback := options
fallback.RoleSoundPresentation = "sound_and_speech"
fallbackSpeech := presentation.DocumentLineSpeechWithOptions(line, fallback)
request.FallbackText, request.FallbackChunks = fallbackSpeech.Text, fallbackSpeech.Chunks
}
if request.Text == "" && len(request.Sound) == 0 {
return
}
speech.Enqueue(request)
}
func document_line_ranges(line *atspi.DocumentLine) []map[string]any {
if line == nil {
return nil
}
ranges := make([]map[string]any, 0, len(line.Contents))
for _, content := range line.Contents {
ranges = append(ranges, map[string]any{"bus": content.Snapshot.BusName,
"source_path": content.Snapshot.Path, "native_text_start": content.Start,
"native_text_end": content.End, "bounds": content.Bounds})
}
return ranges
}
func documentResultMatchesCurrent(result review.Result, current atspi.Snapshot) bool {
if result.DocumentFocus == current.ID() {
return true
}
return result.Moved && result.Err == nil &&
result.DocumentPosition.Snapshot.ID() == current.ID()
}
func presentDocumentResult(logger *slog.Logger, speech speechEnqueuer, result review.Result, options presentation.Options) {
if logger.Enabled(context.Background(), slog.LevelDebug) {
logger.Debug("document navigation completed", "elapsed", result.Elapsed,
"error", result.Err, "document_epoch", result.DocumentEpoch,
"focus", result.DocumentFocus, "document", result.DocumentPosition.Document.ID(),
"application", result.DocumentPosition.Snapshot.ApplicationName, "role", result.DocumentPosition.Snapshot.Role,
"wrapped", result.DocumentWrapped, "navigation", result.DocumentDiagnostics,
"capture_duration", result.CaptureDuration, "capture_reason", result.CaptureReason,
"moved", result.Moved, "target", result.DocumentTarget,
"level", result.DocumentLevel, "backwards", result.DocumentBackwards,
"bus", result.DocumentPosition.Object.ID().BusName,
"source_path", result.DocumentPosition.Object.ID().Path,
"virtual_caret_offset", result.DocumentPosition.Offset,
"caret_sync_error", result.CaretErr,
"sound_geometry_error", result.DocumentSoundErr,
"braille_line_available", result.DocumentBrailleLine != nil,
"braille_read_error", result.DocumentBrailleErr)
}
text, fallbackText, role := "", "", "system"
origin := speechOriginUnknown
var spoken, fallbackSpeech presentation.Speech
var cues []sound.Cue
switch {
case errors.Is(result.Err, atspi.ErrNotInDocument):
text = "Not in web document content."
case errors.Is(result.Err, atspi.ErrProviderHung):
text = "Application is not responding."
case result.Err != nil:
logger.Warn("document navigation failed", "error", result.Err)
text = "Could not find current location."
case !result.Moved:
switch result.DocumentTarget {
case review.DocumentHeading:
if result.DocumentLevel == 0 {
text = "No more headings."
} else {
text = fmt.Sprintf("No more headings at level %d.", result.DocumentLevel)
}
case review.DocumentLink:
text = "No more links."
case review.DocumentButton:
text = "No more buttons."
case review.DocumentCheckBox:
text = "No more check boxes."
case review.DocumentList:
text = "No more lists."
case review.DocumentSeparator:
text = "No more separators."
case review.DocumentBlockQuote:
text = "No more blockquotes."
case review.DocumentComboBox:
text = "No more combo boxes."
case review.DocumentEntry:
text = "No more entries."
case review.DocumentFormField:
text = "No more form fields."
case review.DocumentImage:
text = "No more images."
case review.DocumentLandmark:
text = "No landmark found."
case review.DocumentListItem:
text = "No more list items."
case review.DocumentLiveRegion:
text = "No more live regions."
case review.DocumentParagraph:
text = "No more paragraphs."
case review.DocumentRadioButton:
text = "No more radio buttons."
case review.DocumentTable:
text = "No more tables."
case review.DocumentUnvisitedLink:
text = "No more unvisited links."
case review.DocumentVisitedLink:
text = "No more visited links."
case review.DocumentClickable:
text = "No more clickables."
case review.DocumentChunk:
text = "No more large objects."
}
default:
origin = document_speech_origin(result.DocumentPosition.Snapshot, result.DocumentCaretContext)
if link := result.DocumentPosition.LinkedHeading; link != nil && accessibility_speech_origin(*link) == speechOriginSecret {
origin = speechOriginSecret
}
if link := result.DocumentPosition.LinkedHeading; link != nil {
roleOnly := *link
roleOnly.Name, roleOnly.Text = "", ""
line := atspi.DocumentLine{Contents: []atspi.DocumentContent{
{Snapshot: result.DocumentPosition.Snapshot}, {Snapshot: roleOnly},
}}
spoken = presentation.DocumentLineSpeechWithOptions(line, options)
text = spoken.Text
cues = presentation.DocumentLineSoundCues(line, options)
fallbackOptions := options
fallbackOptions.RoleSoundPresentation = "sound_and_speech"
fallbackSpeech = presentation.DocumentLineSpeechWithOptions(line, fallbackOptions)
fallbackText = fallbackSpeech.Text
} else {
spoken = presentation.FocusSpeechWithOptions(result.DocumentPosition.Snapshot, options)
text = spoken.Text
cues = presentation.SoundCues(result.DocumentPosition.Snapshot, options)
if options.RoleSoundPresentation == "sound_only" {
fallbackOptions := options
fallbackOptions.RoleSoundPresentation = "sound_and_speech"
fallbackSpeech = presentation.FocusSpeechWithOptions(result.DocumentPosition.Snapshot, fallbackOptions)
fallbackText = fallbackSpeech.Text
}
}
if len(cues) > 0 && logger.Enabled(context.Background(), slog.LevelDebug) {
// Record the captured geometry actually used for cue generation.
// Linked headings use the link's geometry independently of the heading.
attributes := []any{
"bus", result.DocumentPosition.Snapshot.BusName,
"source_path", result.DocumentPosition.Snapshot.Path,
"role", result.DocumentPosition.Snapshot.Role,
"spatialize_object_sounds", options.SpatializeObjectSounds,
"sound_geometry", result.DocumentPosition.Snapshot.SoundGeometry,
"cues", cues,
}
if link := result.DocumentPosition.LinkedHeading; link != nil {
attributes = append(attributes,
"linked_heading_bus", link.BusName,
"linked_heading_path", link.Path,
"linked_heading_sound_geometry", link.SoundGeometry)
}
logger.Debug("document navigation sound positioning", attributes...)
}
role = result.DocumentPosition.Snapshot.Role
if result.DocumentWrapped {
if result.DocumentBackwards {
spoken = spoken.Append("Wrapping to bottom.", ". ")
text = spoken.Text
fallbackSpeech = fallbackSpeech.Append("Wrapping to bottom.", ". ")
fallbackText = fallbackSpeech.Text
} else {
spoken = spoken.Append("Wrapping to top.", ". ")
text = spoken.Text
fallbackSpeech = fallbackSpeech.Append("Wrapping to top.", ". ")
fallbackText = fallbackSpeech.Text
}
}
if result.CaretErr != nil {
logger.Warn("document caret synchronization failed", "error", result.CaretErr)
}
}
voiceType := ancestor_voice(result.DocumentCaretContext)
if result.DocumentPosition.LinkedHeading != nil {
voiceType = "hyperlink"
}
ancestors := result.DocumentCaretContext
if len(ancestors) > 0 && ancestors[0].ID() == result.DocumentPosition.Snapshot.ID() {
ancestors = ancestors[1:]
}
speech.Enqueue(speechRequest{origin: origin, Text: text, Chunks: spoken.WithText(text).Chunks, FallbackText: fallbackText, FallbackChunks: fallbackSpeech.WithText(fallbackText).Chunks, Role: role, Sound: cues, Cause: "document-navigation", Context: ancestors, VoiceType: voiceType})
}