277 lines
8.0 KiB
Go
277 lines
8.0 KiB
Go
// Copyright (c) 2026 Storm Dragon
|
|
// Authors: Storm Dragon, Angus Kola
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package app
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.stormux.org/storm/cthulhu/internal/atspi"
|
|
"git.stormux.org/storm/cthulhu/internal/braille"
|
|
"git.stormux.org/storm/cthulhu/internal/config"
|
|
"git.stormux.org/storm/cthulhu/internal/input"
|
|
"git.stormux.org/storm/cthulhu/internal/ocr"
|
|
"git.stormux.org/storm/cthulhu/internal/platform/xorg"
|
|
"git.stormux.org/storm/cthulhu/internal/review"
|
|
)
|
|
|
|
// This joins the actual reader loops, AT-SPI/review, XRecord, native grabs,
|
|
// capture helper and installed engine, with recording output sinks. It does
|
|
// not run an installed reader or establish audible/physical braille acceptance.
|
|
func TestNativeOCRSession(t *testing.T) {
|
|
if os.Getenv("CTHULHU_NATIVE_OCR_SESSION") != "1" {
|
|
t.Skip("requires private Xorg/AT-SPI session and installed Tesseract")
|
|
}
|
|
executable := os.Getenv("CTHULHU_OCR_CAPTURE_BINARY")
|
|
if executable == "" {
|
|
t.Fatal("set CTHULHU_OCR_CAPTURE_BINARY to the temporary reader binary")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
logger := slog.New(slog.DiscardHandler)
|
|
adapter := atspi.New()
|
|
keys := xorg.NewCommands([]string{"Insert"}, input.DesktopKeyboardLayout)
|
|
settings := config.DefaultSettings()
|
|
settings.EnableKeyEcho = false
|
|
settings.KeyBindings = []input.BindingOverride{{Layout: input.DesktopKeyboardLayout, Scope: "global", Command: input.OCRClipboard, Keys: []string{"Cthulhu+F8"}}}
|
|
if err := keys.SetKeyBindings(ctx, settings.KeyBindings); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
listener := xorg.New()
|
|
objectReview, err := review.NewWorker(adapter, time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
output := &ocrTestOutput{speech: make(chan speechRequest, 256), frames: make(chan braille.Frame, 256)}
|
|
gate := newFocusGate(keys)
|
|
live := &liveSettings{}
|
|
live.Store(settings)
|
|
session := new_ocr_session(logger, keys, adapter, objectReview, output, nil, output, gate, live)
|
|
session.capture = ocr.NewCapturer(executable)
|
|
gate.ocr = session
|
|
for _, service := range []interface {
|
|
Start(context.Context) error
|
|
Stop(context.Context) error
|
|
}{keys, adapter, objectReview, listener, session} {
|
|
if err := service.Start(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer service.Stop(context.Background())
|
|
}
|
|
commandDone := make(chan error, 1)
|
|
eventDone := make(chan error, 1)
|
|
inputDone := make(chan struct{})
|
|
go func() {
|
|
commandDone <- (commandRuntime{logger: logger, commands: keys, focusGate: gate, speechOutput: output, objectReview: objectReview, ocr: session, stopRun: cancel}).run(ctx)
|
|
}()
|
|
go func() {
|
|
eventDone <- processEventsWithOCR(ctx, logger, outputQueue{speech: output, braille: output, live: live}, adapter, nil, objectReview, time.Millisecond, live, listener, gate, nil, session)
|
|
}()
|
|
go func() {
|
|
defer close(inputDone)
|
|
processInputEvents(ctx, logger, output, listener, inputEchoSettingsFromConfig(settings), live, keys, session)
|
|
}()
|
|
defer func() {
|
|
cancel()
|
|
if err := <-commandDone; err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := <-eventDone; err != nil {
|
|
t.Error(err)
|
|
}
|
|
<-inputDone
|
|
}()
|
|
fixture := exec.CommandContext(ctx, "/usr/bin/python3", "-u", "../../tests/ocr_session_fixture.py")
|
|
stdin, err := fixture.StdinPipe()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stdout, err := fixture.StdoutPipe()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.Stderr = os.Stderr
|
|
if err = fixture.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { stdin.Close(); fixture.Wait() }()
|
|
scanner := bufio.NewScanner(stdout)
|
|
read := func() map[string]uint64 {
|
|
t.Helper()
|
|
if !scanner.Scan() {
|
|
t.Fatal("OCR fixture did not respond")
|
|
}
|
|
var state map[string]uint64
|
|
if err := json.Unmarshal(scanner.Bytes(), &state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return state
|
|
}
|
|
read()
|
|
action := func(name string) map[string]uint64 {
|
|
t.Helper()
|
|
data, _ := json.Marshal(map[string]string{"action": name})
|
|
if _, err := stdin.Write(append(data, '\n')); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state := read()
|
|
if state["focus"] != state["window"] {
|
|
t.Fatalf("OCR changed native window focus: %+v", state)
|
|
}
|
|
return state
|
|
}
|
|
key := func(chord string) {
|
|
t.Helper()
|
|
if out, err := exec.CommandContext(ctx, "xdotool", "key", "--clearmodifiers", chord).CombinedOutput(); err != nil {
|
|
t.Fatalf("key: %v %s", err, out)
|
|
}
|
|
}
|
|
defer exec.Command("xdotool", "keyup", "Insert", "Control_L", "Shift_L", "o", "Escape").Run()
|
|
wait := func(predicate func() bool) {
|
|
t.Helper()
|
|
for !predicate() {
|
|
select {
|
|
case err := <-session.failures:
|
|
t.Fatal(err)
|
|
case err := <-keys.Failures():
|
|
t.Fatal(err)
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
case <-time.After(time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
focus := func(actionName string) {
|
|
session.mu.Lock()
|
|
previous := session.focus
|
|
session.mu.Unlock()
|
|
action(actionName)
|
|
wait(func() bool {
|
|
session.mu.Lock()
|
|
defer session.mu.Unlock()
|
|
return session.focus.Valid() && session.focus != previous
|
|
})
|
|
}
|
|
// Observe actual focused-widget speech, not merely a valid frame identity.
|
|
for {
|
|
got := ocr_test_receive(t, output.speech)
|
|
if strings.Contains(got.Text, "Painted OCR target") {
|
|
break
|
|
}
|
|
}
|
|
wait(func() bool {
|
|
session.mu.Lock()
|
|
defer session.mu.Unlock()
|
|
return session.focus.Valid()
|
|
})
|
|
expect := func(want string) {
|
|
t.Helper()
|
|
for {
|
|
select {
|
|
case got := <-output.speech:
|
|
t.Logf("output %s: %q", got.Cause, got.Text)
|
|
if got.Cause == "ocr-status" || got.Cause == "ocr-cursor" || got.Cause == "ocr-selection" {
|
|
if got.Text != want {
|
|
t.Fatalf("OCR speech: %+v want %q", got, want)
|
|
}
|
|
return
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
frame := func(text string, cursor int) {
|
|
t.Helper()
|
|
for {
|
|
select {
|
|
case got := <-output.frames:
|
|
if got.Text == text {
|
|
if got.Cursor != cursor || len(got.Routes) != 0 {
|
|
t.Fatalf("OCR braille: %+v", got)
|
|
}
|
|
return
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal(ctx.Err())
|
|
}
|
|
}
|
|
}
|
|
key("Insert+o")
|
|
expect("OCR test window")
|
|
frame("OCR test window", 0)
|
|
wait(keys.OCRReviewEnabled)
|
|
key("shift+Right")
|
|
expect("O selected")
|
|
frame("OCR test window", 1)
|
|
key("shift+Left")
|
|
expect("O unselected")
|
|
frame("OCR test window", 0)
|
|
key("ctrl+Right")
|
|
expect("test")
|
|
frame("OCR test window", 4)
|
|
key("Down")
|
|
expect("Open Settings")
|
|
frame("Open Settings", 4)
|
|
action("state")
|
|
key("Escape")
|
|
expect("OCR dismissed")
|
|
wait(func() bool { return !keys.OCRReviewEnabled() })
|
|
key("Insert+F8")
|
|
expect("OCR test window")
|
|
frame("OCR test window", 0)
|
|
action("clipboard-text")
|
|
key("Down")
|
|
expect("Open Settings")
|
|
key("ctrl+a")
|
|
expect("OCR text selected")
|
|
key("ctrl+c")
|
|
expect("OCR text copied to clipboard")
|
|
session.mu.Lock()
|
|
wantCopy, _ := session.cursor.Slice(0, session.cursor.Length())
|
|
session.mu.Unlock()
|
|
copyResult, err := exec.CommandContext(ctx, "xclip", "-selection", "clipboard", "-out").Output()
|
|
if err != nil || string(copyResult) != wantCopy {
|
|
t.Fatalf("OCR clipboard copy = %q, want %q: %v", copyResult, wantCopy, err)
|
|
}
|
|
// Native focus changes within the same X11 window dismiss a ready result.
|
|
action("other")
|
|
wait(func() bool { return !keys.OCRReviewEnabled() })
|
|
focus("password")
|
|
key("Insert+o")
|
|
expect("OCR test window")
|
|
key("Escape")
|
|
expect("OCR dismissed")
|
|
focus("painted")
|
|
key("Insert+o")
|
|
expect("OCR test window")
|
|
key("h")
|
|
wait(func() bool { return !keys.OCRReviewEnabled() })
|
|
// No accessibility provider exists for this client. The default command,
|
|
// review, refresh and dismissal must work using only X11 ownership/pixels.
|
|
inaccessible := action("inaccessible")
|
|
wait(func() bool { return adapter.Foreground().Client.Window == inaccessible["window"] })
|
|
key("Insert+o")
|
|
expect("OCR test window")
|
|
frame("OCR test window", 0)
|
|
wait(keys.OCRReviewEnabled)
|
|
key("Down")
|
|
expect("Open Settings")
|
|
frame("Open Settings", 0)
|
|
key("Insert+F5")
|
|
expect("OCR text unchanged")
|
|
action("state")
|
|
key("Escape")
|
|
expect("OCR dismissed")
|
|
wait(func() bool { return !keys.OCRReviewEnabled() })
|
|
}
|