232 lines
9.2 KiB
Go
232 lines
9.2 KiB
Go
// Copyright (c) 2026 Storm Dragon
|
|
// Authors: Storm Dragon, Angus Kola
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package ocr
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"image"
|
|
"image/color"
|
|
"os"
|
|
"os/exec"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Run only on a private display with a freshly built helper binary.
|
|
func TestNativeOCRCapture(t *testing.T) {
|
|
if os.Getenv("CTHULHU_NATIVE_OCR_CAPTURE") != "1" {
|
|
t.Skip("private Xvfb fixture only")
|
|
}
|
|
executable := os.Getenv("CTHULHU_OCR_CAPTURE_BINARY")
|
|
if executable == "" {
|
|
t.Fatal("set CTHULHU_OCR_CAPTURE_BINARY to a freshly built reader")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
fixture := exec.CommandContext(ctx, "/usr/bin/python3", "-u", "../../tests/ocr_capture_fixture.py")
|
|
fixture.Env = append(os.Environ(), "NO_AT_BRIDGE=1", "GTK_MODULES=")
|
|
stdin, err := fixture.StdinPipe()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stdout, err := fixture.StdoutPipe()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fixture.Stderr = os.Stderr
|
|
if err = fixture.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = stdin.Close(); _ = fixture.Wait() }()
|
|
scanner := bufio.NewScanner(stdout)
|
|
if !scanner.Scan() {
|
|
t.Fatal("fixture did not start")
|
|
}
|
|
var windows map[string]uint64
|
|
if err = json.Unmarshal(scanner.Bytes(), &windows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
command := func(command map[string]any) map[string]uint64 {
|
|
t.Helper()
|
|
data, _ := json.Marshal(command)
|
|
if _, err := stdin.Write(append(data, '\n')); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !scanner.Scan() {
|
|
t.Fatal("fixture did not respond")
|
|
}
|
|
var state map[string]uint64
|
|
if err := json.Unmarshal(scanner.Bytes(), &state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return state
|
|
}
|
|
state := func() map[string]uint64 { return command(map[string]any{"action": "state"}) }
|
|
capturer := NewCapturer(executable)
|
|
request := CaptureRequest{Kind: WindowSource, Foreground: windows["window"], Generation: 1}
|
|
capture := func(request CaptureRequest) Capture {
|
|
t.Helper()
|
|
before := state()
|
|
result, err := capturer.Capture(ctx, request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after := state()
|
|
if before["focus"] != after["focus"] || before["windows"] != after["windows"] {
|
|
t.Fatalf("capture changed native focus/windows: %v -> %v", before, after)
|
|
}
|
|
return result
|
|
}
|
|
expectPixel := func(result Capture, point image.Point, want color.RGBA) {
|
|
t.Helper()
|
|
got := color.RGBAModel.Convert(result.Pixels.At(point.X, point.Y)).(color.RGBA)
|
|
if got != want {
|
|
t.Fatalf("pixel %v = %v, want %v", point, got, want)
|
|
}
|
|
}
|
|
// XGetImage uses physical pixels at both GDK scales. Provider conversion is
|
|
// intentionally a separate target-owner gate, not established by this test.
|
|
for _, scale := range []string{"1", "2"} {
|
|
t.Setenv("GDK_SCALE", scale)
|
|
result := capture(request)
|
|
if result.ScreenBounds != image.Rect(40, 30, 340, 230) || result.Pixels.Bounds() != image.Rect(0, 0, 300, 200) {
|
|
t.Fatalf("scale %s capture: %+v", scale, result.CaptureHeader)
|
|
}
|
|
expectPixel(result, image.Pt(0, 0), color.RGBA{0x33, 0x66, 0x99, 255})
|
|
expectPixel(result, image.Pt(10, 15), color.RGBA{0xab, 0xcd, 0xef, 255})
|
|
mapped, ok := result.Transform.ScreenRectangle(image.Rect(10, 15, 20, 25))
|
|
if !ok || mapped != image.Rect(50, 45, 60, 55) {
|
|
t.Fatalf("pixel mapping: %v, %v", mapped, ok)
|
|
}
|
|
}
|
|
t.Setenv("GDK_SCALE", "1")
|
|
// A usable X11 source must not be rejected solely by the session label.
|
|
// This tests policy on Xvfb, not capture on a real Wayland compositor.
|
|
t.Setenv("XDG_SESSION_TYPE", "wayland")
|
|
expectPixel(capture(request), image.Pt(0, 0), color.RGBA{0x33, 0x66, 0x99, 255})
|
|
t.Setenv("XDG_SESSION_TYPE", "x11")
|
|
childRequest := request
|
|
childRequest.Window = windows["child"]
|
|
child := capture(childRequest)
|
|
if child.ScreenBounds != image.Rect(50, 45, 130, 95) {
|
|
t.Fatalf("child capture: %+v", child.CaptureHeader)
|
|
}
|
|
expectPixel(child, image.Pt(0, 0), color.RGBA{0xab, 0xcd, 0xef, 255})
|
|
objectRequest := request
|
|
objectRequest.Kind, objectRequest.Bounds = ObjectSource, image.Rect(-10, -10, 90, 100)
|
|
if result := capture(objectRequest); result.ScreenBounds != image.Rect(40, 30, 90, 100) {
|
|
t.Fatalf("object clip: %+v", result.CaptureHeader)
|
|
}
|
|
objectRequest.Bounds = image.Rect(-100, -100, -1, -1)
|
|
if _, err := capturer.Capture(ctx, objectRequest); !errors.Is(err, ErrSource) {
|
|
t.Fatalf("offscreen source: %v", err)
|
|
}
|
|
desktopRequest := request
|
|
desktopRequest.Kind = DesktopSource
|
|
desktop := capture(desktopRequest)
|
|
if desktop.ScreenBounds != desktop.RootBounds || desktop.SourceWindow != 0 {
|
|
t.Fatalf("desktop: %+v", desktop.CaptureHeader)
|
|
}
|
|
expectPixel(desktop, image.Pt(350, 30), color.RGBA{0x99, 0x22, 0x44, 255})
|
|
if os.Getenv("CTHULHU_NATIVE_OCR") == "1" {
|
|
command(map[string]any{"action": "activate", "window": "painted"})
|
|
paintedRequest := request
|
|
paintedRequest.Foreground = windows["painted"]
|
|
painted := capture(paintedRequest)
|
|
result, err := NewEngine().Recognize(ctx, painted.Pixels, Options{})
|
|
if err != nil || result.Text != "OCR test window\nOpen Settings\nValue 42" {
|
|
t.Fatalf("captured X11 recognition: %q, %v", result.Text, err)
|
|
}
|
|
word, ok := result.WordAt(0)
|
|
mapped, mappedOK := painted.Transform.ScreenRectangle(word.Bounds)
|
|
if !ok || !mappedOK || !mapped.In(painted.ScreenBounds) {
|
|
t.Fatalf("recognized screen word mapping: %v, %v", mapped, mappedOK)
|
|
}
|
|
command(map[string]any{"action": "activate"})
|
|
}
|
|
clipboardRequest := request
|
|
clipboardRequest.Kind = ClipboardSource
|
|
clipboard := capture(clipboardRequest)
|
|
if clipboard.Transform.HasScreenMapping || clipboard.Pixels.Bounds() != image.Rect(0, 0, 640, 220) {
|
|
t.Fatalf("clipboard mapping: %+v", clipboard.CaptureHeader)
|
|
}
|
|
command(map[string]any{"action": "clipboard-text"})
|
|
if _, err := capturer.Capture(ctx, clipboardRequest); !errors.Is(err, ErrSource) {
|
|
t.Fatalf("text clipboard captured as image: %v", err)
|
|
}
|
|
// The earlier snapshot survives clipboard replacement and is recognized from
|
|
// those owned pixels rather than fetching the current clipboard again.
|
|
if os.Getenv("CTHULHU_NATIVE_OCR") == "1" {
|
|
result, err := NewEngine().Recognize(ctx, clipboard.Pixels, Options{})
|
|
if err != nil || result.Text != "OCR test window\nOpen Settings\nValue 42" {
|
|
t.Fatalf("captured clipboard recognition: %q, %v", result.Text, err)
|
|
}
|
|
}
|
|
// Synchronize on the real selection request, switch foreground during the
|
|
// native transfer, and release pixels. The helper must reject that result.
|
|
command(map[string]any{"action": "clipboard-block"})
|
|
completed := make(chan error, 1)
|
|
go func() { _, err := capturer.Capture(ctx, clipboardRequest); completed <- err }()
|
|
if !scanner.Scan() || scanner.Text() != `{"capture-request": 1}` {
|
|
t.Fatal("clipboard capture did not request its pixels")
|
|
}
|
|
command(map[string]any{"action": "activate", "window": "other"})
|
|
command(map[string]any{"action": "clipboard-release"})
|
|
if err := <-completed; !errors.Is(err, ErrSourceChanged) {
|
|
t.Fatalf("foreground change during capture: %v", err)
|
|
}
|
|
command(map[string]any{"action": "activate"})
|
|
go func() { _, err := capturer.Capture(ctx, clipboardRequest); completed <- err }()
|
|
if !scanner.Scan() || scanner.Text() != `{"capture-request": 1}` {
|
|
t.Fatal("geometry-change capture did not start")
|
|
}
|
|
command(map[string]any{"action": "resize", "width": 310, "height": 210})
|
|
command(map[string]any{"action": "clipboard-release"})
|
|
if err := <-completed; !errors.Is(err, ErrSourceChanged) {
|
|
t.Fatalf("geometry change during capture: %v", err)
|
|
}
|
|
command(map[string]any{"action": "resize", "width": 300, "height": 200})
|
|
// A provider that never replies must not hold the reader's worker forever.
|
|
go func() { _, err := capturer.Capture(ctx, clipboardRequest); completed <- err }()
|
|
if !scanner.Scan() || scanner.Text() != `{"capture-request": 1}` {
|
|
t.Fatal("blocked clipboard capture did not start")
|
|
}
|
|
if err := <-completed; !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("blocked native clipboard deadline: %v", err)
|
|
}
|
|
command(map[string]any{"action": "clipboard-text"})
|
|
command(map[string]any{"action": "overlay"})
|
|
expectPixel(capture(request), image.Pt(0, 0), color.RGBA{0xaa, 0x11, 0, 255})
|
|
command(map[string]any{"action": "move", "x": -20, "y": -10})
|
|
if result := capture(request); result.ScreenBounds != image.Rect(0, 0, 280, 190) || result.SourceBounds != image.Rect(-20, -10, 280, 190) {
|
|
t.Fatalf("negative window clip: %+v", result.CaptureHeader)
|
|
}
|
|
command(map[string]any{"action": "activate", "window": "other"})
|
|
if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrSourceChanged) {
|
|
t.Fatalf("foreground mismatch: %v", err)
|
|
}
|
|
childRequest.Foreground = windows["other"]
|
|
childRequest.Window = windows["child"]
|
|
if _, err := capturer.Capture(ctx, childRequest); !errors.Is(err, ErrSource) {
|
|
t.Fatalf("unrelated window accepted: %v", err)
|
|
}
|
|
command(map[string]any{"action": "activate"})
|
|
command(map[string]any{"action": "hide"})
|
|
if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrSource) {
|
|
t.Fatalf("hidden window: %v", err)
|
|
}
|
|
command(map[string]any{"action": "destroy"})
|
|
if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrSource) {
|
|
t.Fatalf("destroyed window: %v", err)
|
|
}
|
|
t.Setenv("DISPLAY", "")
|
|
if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrCapture) {
|
|
t.Fatalf("missing display: %v", err)
|
|
}
|
|
}
|