// Copyright (c) 2026 Storm Dragon // Authors: Storm Dragon, Angus Kola // SPDX-License-Identifier: GPL-3.0-or-later package ocr import ( "bufio" "context" "encoding/json" "errors" "image" "image/color" "os" "os/exec" "testing" "time" ) // Run only on a private display with a freshly built helper binary. func TestNativeOCRCapture(t *testing.T) { if os.Getenv("CTHULHU_NATIVE_OCR_CAPTURE") != "1" { t.Skip("private Xvfb fixture only") } executable := os.Getenv("CTHULHU_OCR_CAPTURE_BINARY") if executable == "" { t.Fatal("set CTHULHU_OCR_CAPTURE_BINARY to a freshly built reader") } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() fixture := exec.CommandContext(ctx, "/usr/bin/python3", "-u", "../../tests/ocr_capture_fixture.py") fixture.Env = append(os.Environ(), "NO_AT_BRIDGE=1", "GTK_MODULES=") stdin, err := fixture.StdinPipe() if err != nil { t.Fatal(err) } stdout, err := fixture.StdoutPipe() if err != nil { t.Fatal(err) } fixture.Stderr = os.Stderr if err = fixture.Start(); err != nil { t.Fatal(err) } defer func() { _ = stdin.Close(); _ = fixture.Wait() }() scanner := bufio.NewScanner(stdout) if !scanner.Scan() { t.Fatal("fixture did not start") } var windows map[string]uint64 if err = json.Unmarshal(scanner.Bytes(), &windows); err != nil { t.Fatal(err) } command := func(command map[string]any) map[string]uint64 { t.Helper() data, _ := json.Marshal(command) if _, err := stdin.Write(append(data, '\n')); err != nil { t.Fatal(err) } if !scanner.Scan() { t.Fatal("fixture did not respond") } var state map[string]uint64 if err := json.Unmarshal(scanner.Bytes(), &state); err != nil { t.Fatal(err) } return state } state := func() map[string]uint64 { return command(map[string]any{"action": "state"}) } capturer := NewCapturer(executable) request := CaptureRequest{Kind: WindowSource, Foreground: windows["window"], Generation: 1} capture := func(request CaptureRequest) Capture { t.Helper() before := state() result, err := capturer.Capture(ctx, request) if err != nil { t.Fatal(err) } after := state() if before["focus"] != after["focus"] || before["windows"] != after["windows"] { t.Fatalf("capture changed native focus/windows: %v -> %v", before, after) } return result } expectPixel := func(result Capture, point image.Point, want color.RGBA) { t.Helper() got := color.RGBAModel.Convert(result.Pixels.At(point.X, point.Y)).(color.RGBA) if got != want { t.Fatalf("pixel %v = %v, want %v", point, got, want) } } // XGetImage uses physical pixels at both GDK scales. Provider conversion is // intentionally a separate target-owner gate, not established by this test. for _, scale := range []string{"1", "2"} { t.Setenv("GDK_SCALE", scale) result := capture(request) if result.ScreenBounds != image.Rect(40, 30, 340, 230) || result.Pixels.Bounds() != image.Rect(0, 0, 300, 200) { t.Fatalf("scale %s capture: %+v", scale, result.CaptureHeader) } expectPixel(result, image.Pt(0, 0), color.RGBA{0x33, 0x66, 0x99, 255}) expectPixel(result, image.Pt(10, 15), color.RGBA{0xab, 0xcd, 0xef, 255}) mapped, ok := result.Transform.ScreenRectangle(image.Rect(10, 15, 20, 25)) if !ok || mapped != image.Rect(50, 45, 60, 55) { t.Fatalf("pixel mapping: %v, %v", mapped, ok) } } t.Setenv("GDK_SCALE", "1") // A usable X11 source must not be rejected solely by the session label. // This tests policy on Xvfb, not capture on a real Wayland compositor. t.Setenv("XDG_SESSION_TYPE", "wayland") expectPixel(capture(request), image.Pt(0, 0), color.RGBA{0x33, 0x66, 0x99, 255}) t.Setenv("XDG_SESSION_TYPE", "x11") childRequest := request childRequest.Window = windows["child"] child := capture(childRequest) if child.ScreenBounds != image.Rect(50, 45, 130, 95) { t.Fatalf("child capture: %+v", child.CaptureHeader) } expectPixel(child, image.Pt(0, 0), color.RGBA{0xab, 0xcd, 0xef, 255}) objectRequest := request objectRequest.Kind, objectRequest.Bounds = ObjectSource, image.Rect(-10, -10, 90, 100) if result := capture(objectRequest); result.ScreenBounds != image.Rect(40, 30, 90, 100) { t.Fatalf("object clip: %+v", result.CaptureHeader) } objectRequest.Bounds = image.Rect(-100, -100, -1, -1) if _, err := capturer.Capture(ctx, objectRequest); !errors.Is(err, ErrSource) { t.Fatalf("offscreen source: %v", err) } desktopRequest := request desktopRequest.Kind = DesktopSource desktop := capture(desktopRequest) if desktop.ScreenBounds != desktop.RootBounds || desktop.SourceWindow != 0 { t.Fatalf("desktop: %+v", desktop.CaptureHeader) } expectPixel(desktop, image.Pt(350, 30), color.RGBA{0x99, 0x22, 0x44, 255}) if os.Getenv("CTHULHU_NATIVE_OCR") == "1" { command(map[string]any{"action": "activate", "window": "painted"}) paintedRequest := request paintedRequest.Foreground = windows["painted"] painted := capture(paintedRequest) result, err := NewEngine().Recognize(ctx, painted.Pixels, Options{}) if err != nil || result.Text != "OCR test window\nOpen Settings\nValue 42" { t.Fatalf("captured X11 recognition: %q, %v", result.Text, err) } word, ok := result.WordAt(0) mapped, mappedOK := painted.Transform.ScreenRectangle(word.Bounds) if !ok || !mappedOK || !mapped.In(painted.ScreenBounds) { t.Fatalf("recognized screen word mapping: %v, %v", mapped, mappedOK) } command(map[string]any{"action": "activate"}) } clipboardRequest := request clipboardRequest.Kind = ClipboardSource clipboard := capture(clipboardRequest) if clipboard.Transform.HasScreenMapping || clipboard.Pixels.Bounds() != image.Rect(0, 0, 640, 220) { t.Fatalf("clipboard mapping: %+v", clipboard.CaptureHeader) } command(map[string]any{"action": "clipboard-text"}) if _, err := capturer.Capture(ctx, clipboardRequest); !errors.Is(err, ErrSource) { t.Fatalf("text clipboard captured as image: %v", err) } // The earlier snapshot survives clipboard replacement and is recognized from // those owned pixels rather than fetching the current clipboard again. if os.Getenv("CTHULHU_NATIVE_OCR") == "1" { result, err := NewEngine().Recognize(ctx, clipboard.Pixels, Options{}) if err != nil || result.Text != "OCR test window\nOpen Settings\nValue 42" { t.Fatalf("captured clipboard recognition: %q, %v", result.Text, err) } } // Synchronize on the real selection request, switch foreground during the // native transfer, and release pixels. The helper must reject that result. command(map[string]any{"action": "clipboard-block"}) completed := make(chan error, 1) go func() { _, err := capturer.Capture(ctx, clipboardRequest); completed <- err }() if !scanner.Scan() || scanner.Text() != `{"capture-request": 1}` { t.Fatal("clipboard capture did not request its pixels") } command(map[string]any{"action": "activate", "window": "other"}) command(map[string]any{"action": "clipboard-release"}) if err := <-completed; !errors.Is(err, ErrSourceChanged) { t.Fatalf("foreground change during capture: %v", err) } command(map[string]any{"action": "activate"}) go func() { _, err := capturer.Capture(ctx, clipboardRequest); completed <- err }() if !scanner.Scan() || scanner.Text() != `{"capture-request": 1}` { t.Fatal("geometry-change capture did not start") } command(map[string]any{"action": "resize", "width": 310, "height": 210}) command(map[string]any{"action": "clipboard-release"}) if err := <-completed; !errors.Is(err, ErrSourceChanged) { t.Fatalf("geometry change during capture: %v", err) } command(map[string]any{"action": "resize", "width": 300, "height": 200}) // A provider that never replies must not hold the reader's worker forever. go func() { _, err := capturer.Capture(ctx, clipboardRequest); completed <- err }() if !scanner.Scan() || scanner.Text() != `{"capture-request": 1}` { t.Fatal("blocked clipboard capture did not start") } if err := <-completed; !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("blocked native clipboard deadline: %v", err) } command(map[string]any{"action": "clipboard-text"}) command(map[string]any{"action": "overlay"}) expectPixel(capture(request), image.Pt(0, 0), color.RGBA{0xaa, 0x11, 0, 255}) command(map[string]any{"action": "move", "x": -20, "y": -10}) if result := capture(request); result.ScreenBounds != image.Rect(0, 0, 280, 190) || result.SourceBounds != image.Rect(-20, -10, 280, 190) { t.Fatalf("negative window clip: %+v", result.CaptureHeader) } command(map[string]any{"action": "activate", "window": "other"}) if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrSourceChanged) { t.Fatalf("foreground mismatch: %v", err) } childRequest.Foreground = windows["other"] childRequest.Window = windows["child"] if _, err := capturer.Capture(ctx, childRequest); !errors.Is(err, ErrSource) { t.Fatalf("unrelated window accepted: %v", err) } command(map[string]any{"action": "activate"}) command(map[string]any{"action": "hide"}) if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrSource) { t.Fatalf("hidden window: %v", err) } command(map[string]any{"action": "destroy"}) if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrSource) { t.Fatalf("destroyed window: %v", err) } t.Setenv("DISPLAY", "") if _, err := capturer.Capture(ctx, request); !errors.Is(err, ErrCapture) { t.Fatalf("missing display: %v", err) } }