// Copyright (c) 2026 Storm Dragon // Authors: Storm Dragon, Angus Kola // SPDX-License-Identifier: GPL-3.0-or-later package app import ( "context" "errors" "image" "log/slog" "os" "sync" "time" "unicode/utf8" "git.stormux.org/storm/cthulhu/internal/atspi" "git.stormux.org/storm/cthulhu/internal/braille" "git.stormux.org/storm/cthulhu/internal/config" "git.stormux.org/storm/cthulhu/internal/input" "git.stormux.org/storm/cthulhu/internal/ocr" "git.stormux.org/storm/cthulhu/internal/review" ) type ocrKeyOwner interface { SetOCRReview(context.Context, bool) error SetOCRContext(context.Context, uint64, bool) error } type ocrClicker interface { ClickOCR(context.Context, uint64, image.Rectangle, image.Point) error } type ocrConfirmer interface { Confirm(context.Context, string) (bool, error) } type ocrGeometrySource interface { CaptureOCRGeometry(context.Context, atspi.ObjectID, *atspi.Rectangle) (atspi.OCRGeometry, error) } type ocrInterestSource interface { ObjectOfInterest(context.Context) (review.Interest, error) } type ocrCaptureSource interface { Capture(context.Context, ocr.CaptureRequest) (ocr.Capture, error) } type ocrRecognizer interface { Recognize(context.Context, image.Image, ocr.Options) (ocr.Result, error) } type ocrReadAllStarter interface { BeginReadAll(context.Context, uint32) context.Context } type ocrJob struct { window atspi.ObjectID ctx context.Context generation uint64 focus atspi.ObjectID foreground atspi.ForegroundState kind ocr.SourceKind options ocr.Options source review.Interest snapshot ocr.Capture refresh bool automatic bool autoReadAll bool copyFirstResult bool timestamp uint32 } // ocrSession owns source/job/result generations and the virtual cursor under mu. // A single worker serializes capture, recognition and acknowledged input changes. // Retirement cancels synchronously; the worker joins children and releases native // input before admitting the next job. No native work runs on an input/event loop. type ocrSession struct { window atspi.ObjectID mu sync.Mutex logger *slog.Logger keys ocrKeyOwner geometry ocrGeometrySource interest ocrInterestSource foreground foregroundMonitor capture ocrCaptureSource engine ocrRecognizer copier ocrTextCopier copyRequests chan ocrCopyRequest activateRequests chan ocrActivateRequest clicker ocrClicker confirmer ocrConfirmer speech speechEnqueuer readAll ocrReadAllStarter readAllContext context.Context // Current speech request; cleared on retirement/refresh. readAllDone chan error display brailleFrameEnqueuer gate *focusGate live *liveSettings ctx context.Context stop context.CancelFunc done chan struct{} wake chan struct{} failures chan error focus atspi.ObjectID focusForeground uint64 job ocrJob cancel context.CancelFunc generation uint64 cursor *ocr.Cursor selectionAnchor int // -1 means no active virtual selection. source review.Interest pixels ocr.Capture held map[string]bool cutoff uint32 refreshing bool refreshFailed bool autoTimer *time.Timer activating bool activationInterrupted bool } func new_ocr_session(logger *slog.Logger, commands commandListener, adapter Adapter, interest *review.Worker, speech speechEnqueuer, readAll ocrReadAllStarter, display brailleFrameEnqueuer, gate *focusGate, live *liveSettings) *ocrSession { executable, _ := os.Executable() session := &ocrSession{logger: logger, interest: interest, speech: gatedSpeech{speech: speech, gate: gate}, readAll: readAll, display: display, gate: gate, live: live, capture: ocr.NewCapturer(executable), engine: ocr.NewEngine(), copier: xclipOCRCopier{}, copyRequests: make(chan ocrCopyRequest, 1), activateRequests: make(chan ocrActivateRequest, 1), confirmer: ocrDialogConfirmer{executable: executable}, wake: make(chan struct{}, 1), failures: make(chan error, 1), held: make(map[string]bool), selectionAnchor: -1} session.keys, _ = commands.(ocrKeyOwner) session.clicker, _ = commands.(ocrClicker) session.geometry, _ = adapter.(ocrGeometrySource) session.foreground, _ = adapter.(foregroundMonitor) return session } func (session *ocrSession) Start(ctx context.Context) error { session.mu.Lock() session.ctx, session.stop = context.WithCancel(ctx) session.done = make(chan struct{}) session.mu.Unlock() go session.run() return nil } func (session *ocrSession) Stop(ctx context.Context) error { session.Retire() session.mu.Lock() stop, done := session.stop, session.done session.mu.Unlock() if stop == nil { return nil } stop() select { case <-done: return nil case <-ctx.Done(): return ctx.Err() } } func (session *ocrSession) signal() { select { case session.wake <- struct{}{}: default: } } func (session *ocrSession) retire_locked() { if session.autoTimer != nil { session.autoTimer.Stop() session.autoTimer = nil } if session.cancel == nil { return } session.logger.Debug("OCR session retired", "generation", session.generation) session.cancel() session.cancel = nil session.generation++ session.cursor = nil session.selectionAnchor = -1 session.source = review.Interest{} session.pixels = ocr.Capture{} session.refreshing = false session.refreshFailed = false session.readAllContext, session.readAllDone = nil, nil session.activating = false session.activationInterrupted = false if session.display != nil { session.display.EnqueueFrame(braille.Frame{Cursor: -1}) } session.signal() } func (session *ocrSession) Retire() { if session == nil { return } session.mu.Lock() defer session.mu.Unlock() session.retire_locked() } func (session *ocrSession) Secure() { if session == nil { return } session.mu.Lock() defer session.mu.Unlock() session.retire_locked() session.focus, session.window = atspi.ObjectID{}, atspi.ObjectID{} session.focusForeground = 0 clear(session.held) } func ocr_source_command(command input.Command) (ocr.SourceKind, bool) { switch command { case input.OCRObject: return ocr.ObjectSource, true case input.OCRWindow: return ocr.WindowSource, true case input.OCRDesktop: return ocr.DesktopSource, true case input.OCRClipboard: return ocr.ClipboardSource, true } return "", false } func (session *ocrSession) request_locked(kind ocr.SourceKind, timestamp uint32) { session.retire_locked() session.cutoff = timestamp if session.ctx == nil || session.ctx.Err() != nil { return } if session.gate.Locked() || session.gate.Suspended() { return } if session.keys == nil || session.foreground == nil { session.status_locked("OCR source unavailable") return } foreground := session.foreground.Foreground() if !foreground.Client.Known || foreground.Client.Window == 0 { session.status_locked("OCR source unavailable") return } // An inaccessible client may leave the previous application's AT-SPI focus // behind. It must not supply object geometry or invalidate this client's job. if session.focusForeground != foreground.Client.Window { session.focus, session.window = atspi.ObjectID{}, atspi.ObjectID{} } if kind == ocr.ObjectSource && (session.geometry == nil || session.interest == nil || !session.focus.Valid()) { session.status_locked("OCR source unavailable") return } session.generation++ ctx, cancel := context.WithCancel(session.ctx) session.cancel = cancel settings := session.settings().OCR session.job = ocrJob{ctx: ctx, generation: session.generation, window: session.window, focus: session.focus, foreground: foreground, kind: kind, autoReadAll: settings.AutoReadAll, copyFirstResult: settings.CopyFirstResult, timestamp: timestamp, options: ocr.Options{Language: settings.Language, Layout: ocr.Layout(settings.Layout), Scale: settings.Scale, Grayscale: settings.Grayscale, Invert: settings.Invert, BlackWhite: settings.BlackWhite, Threshold: settings.Threshold}} session.signal() } func (session *ocrSession) refresh_locked(automatic bool) { if session.cancel == nil || session.cursor == nil || session.refreshing || !session.current_locked(session.job) { return } if automatic && (session.job.kind == ocr.ClipboardSource || !session.settings().OCR.AutoRefresh) { return } // Keep a periodic capture from truncating paced reading. Completion and // interruption are observed without waiting on the speech worker. if automatic && session.readAllContext != nil && session.readAllContext.Err() == nil { select { case <-session.readAllDone: default: session.schedule_refresh_locked() return } } session.readAllContext, session.readAllDone = nil, nil if session.autoTimer != nil { session.autoTimer.Stop() session.autoTimer = nil } session.cancel() session.generation++ ctx, cancel := context.WithCancel(session.ctx) session.cancel = cancel old := session.job settings := session.settings().OCR session.job = ocrJob{ctx: ctx, generation: session.generation, window: old.window, focus: old.focus, foreground: old.foreground, kind: old.kind, source: session.source, snapshot: session.pixels, refresh: true, automatic: automatic, options: ocr.Options{Language: settings.Language, Layout: ocr.Layout(settings.Layout), Scale: settings.Scale, Grayscale: settings.Grayscale, Invert: settings.Invert, BlackWhite: settings.BlackWhite, Threshold: settings.Threshold}} session.refreshing = true session.refreshFailed = true session.signal() } func (session *ocrSession) schedule_refresh_locked() { if session.autoTimer != nil { session.autoTimer.Stop() session.autoTimer = nil } settings := session.settings().OCR if settings.AutoRefresh && session.cursor != nil && session.job.kind != ocr.ClipboardSource && !session.refreshFailed { session.autoTimer = time.NewTimer(time.Duration(settings.RefreshInterval) * time.Millisecond) } } func (session *ocrSession) timer_channel() <-chan time.Time { session.mu.Lock() defer session.mu.Unlock() if session.autoTimer != nil { return session.autoTimer.C } return nil } func (session *ocrSession) run() { defer close(session.done) for { select { case <-session.ctx.Done(): session.Retire() session.release_input() return case <-session.wake: session.mu.Lock() job, active, ready := session.job, session.cancel != nil, session.cursor != nil && !session.refreshing session.mu.Unlock() if !active { session.release_input() continue } if !ready { session.recognize(job) } case request := <-session.copyRequests: session.copy(request) case request := <-session.activateRequests: session.activate(request) case <-session.timer_channel(): session.mu.Lock() session.autoTimer = nil session.refresh_locked(true) session.mu.Unlock() } } } func (session *ocrSession) release_input() { if session.keys == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := session.keys.SetOCRReview(ctx, false); err != nil && session.ctx.Err() == nil { if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { // A physical key may still be held. Retry after the chord releases; // waiting for that release is not a failed native grab transaction. session.signal() return } // Failed release must stop the reader rather than leave swallowed keys. select { case session.failures <- err: default: } } } func (session *ocrSession) current_locked(job ocrJob) bool { return session.cancel != nil && session.generation == job.generation && job.ctx.Err() == nil && !session.gate.Locked() && !session.gate.Suspended() && session.focus == job.focus && session.foreground.Foreground() == job.foreground } func (session *ocrSession) recognize(job ocrJob) { var err error if !job.refresh { err = session.keys.SetOCRContext(job.ctx, job.foreground.Client.Window, true) } interest := job.source var capture ocr.Capture var result ocr.Result // Only object capture needs AT-SPI geometry. Window, desktop and clipboard // jobs belong to the X11 foreground even when it has no accessible objects. request := ocr.CaptureRequest{Kind: job.kind, Foreground: job.foreground.Client.Window, Generation: job.generation} if err == nil && job.kind == ocr.ObjectSource { if !job.refresh { interest, err = session.interest.ObjectOfInterest(job.ctx) } if err == nil { var zone *atspi.Rectangle if interest.HasZone { zone = &interest.Zone } var geometry atspi.OCRGeometry geometry, err = session.geometry.CaptureOCRGeometry(job.ctx, interest.Object, zone) if err == nil && geometry.Foreground != job.foreground { err = ocr.ErrSourceChanged } request.Bounds = geometry.ScreenBounds if err == nil { session.mu.Lock() if session.current_locked(job) { session.source = interest } else { err = context.Canceled } session.mu.Unlock() } } } if err == nil { if job.refresh && job.kind == ocr.ClipboardSource { capture = job.snapshot } else { capture, err = session.capture.Capture(job.ctx, request) } } if err == nil { result, err = session.engine.Recognize(job.ctx, capture.Pixels, job.options) if err == nil && result.Width > 0 && result.Height > 0 { capture.Transform.ImageContent = image.Rect(0, 0, result.Width, result.Height) } } if err == nil { var fresh atspi.OCRGeometry if err == nil && job.kind == ocr.ObjectSource { var zone *atspi.Rectangle if interest.HasZone { zone = &interest.Zone } fresh, err = session.geometry.CaptureOCRGeometry(job.ctx, interest.Object, zone) if err == nil && (fresh.Foreground != job.foreground || fresh.ScreenBounds != request.Bounds) { err = ocr.ErrSourceChanged } } if err == nil && result.Text != "" && !job.refresh { err = session.keys.SetOCRContext(job.ctx, job.foreground.Client.Window, false) } } session.mu.Lock() current := session.current_locked(job) if current { if job.refresh { session.finish_refresh_locked(job, interest, capture, result, err) } else if err != nil || result.Text == "" { session.retire_locked() message := "OCR recognition failed" switch { case errors.Is(err, ocr.ErrUnavailable): message = "OCR not available" case err == nil: message = "No text recognized" case errors.Is(err, ocr.ErrSourceChanged): message = "OCR source changed" case errors.Is(err, atspi.ErrOCRGeometry), errors.Is(err, atspi.ErrObjectNotFound), errors.Is(err, review.ErrNoCurrent), errors.Is(err, ocr.ErrSource), errors.Is(err, ocr.ErrCapture): message = "OCR source unavailable" } session.status_locked(message) } else { session.source, session.pixels = interest, capture session.cursor = ocr.NewCursor(result) session.selectionAnchor = -1 if job.autoReadAll && session.readAll != nil { if session.display != nil { session.display.EnqueueFrame(session.frame_locked(session.cursor.Current(ocr.LineUnit))) } session.read_all_locked(job.timestamp) } else { session.present_locked(session.cursor.Current(ocr.LineUnit), ocr.LineUnit) } session.schedule_refresh_locked() if job.copyFirstResult && session.copier != nil && session.copyRequests != nil { select { case session.copyRequests <- ocrCopyRequest{ctx: job.ctx, generation: job.generation, text: result.Text, automatic: true}: default: } } } } else if session.generation == job.generation { session.retire_locked() } session.mu.Unlock() if !job.refresh && (!current || err != nil || result.Text == "") { session.release_input() } } func (session *ocrSession) status_locked(message string) { if session.gate.Locked() || session.gate.Suspended() { return } session.speech.Enqueue(speechRequest{Text: message, Role: "system", Cause: "ocr-status"}) if session.display != nil { session.display.EnqueueFrame(braille.Frame{Text: message, Cursor: -1}) } session.logger.Debug("OCR status", "text", message) } func (session *ocrSession) present_locked(position ocr.Position, unit ocr.Unit) { session.present_spelling_locked(position, unit, review.FlatSpeak) } func (session *ocrSession) present_spelling_locked(position ocr.Position, unit ocr.Unit, style review.FlatSpeech) { text := presentationOCRText(position.Text, unit) request := speechRequest{Text: text, Role: "text", Cause: "ocr-cursor", context: session.job.ctx, Character: unit == ocr.Character || unit == ocr.LineStart || unit == ocr.LineEnd} set_review_spelling(&request, position.Text, style) session.present_request_locked(position, unit, request) } func (session *ocrSession) present_request_locked(position ocr.Position, unit ocr.Unit, request speechRequest) { session.speech.Enqueue(request) frame := session.frame_locked(position) if session.display != nil { session.display.EnqueueFrame(frame) } from, to := session.selection_locked() session.logger.Debug("OCR cursor presentation", "generation", session.generation, "source_path", session.source.Object.Path, "focus_path", session.focus.Path, "unit", unit, "offset", position.Offset, "line_start", position.LineStart, "selection_start", from, "selection_end", to, "speech_text", request.Text, "spelling", request.Spelling, "braille_text", frame.Text, "braille_cursor", frame.Cursor) } func (session *ocrSession) selection_locked() (int, int) { if session.cursor == nil || session.selectionAnchor < 0 { return -1, -1 } return min(session.selectionAnchor, session.cursor.Offset()), max(session.selectionAnchor, session.cursor.Offset()) } func (session *ocrSession) frame_locked(position ocr.Position) braille.Frame { frame := ocr_braille_frame(position) start := position.LineStart + position.Cursor - frame.Cursor length := utf8.RuneCountInString(frame.Text) frame.Virtual = &braille.VirtualRoute{Generation: session.generation, Start: 0, End: length, SourceStart: start} if from, to := session.selection_locked(); from >= 0 && from != to { indicator := byte(session.settings().Braille.SelectionIndicator) if indicator != 0 { frame.Dots = make([]byte, length) for offset := max(from, start); offset < min(to, start+length); offset++ { frame.Dots[offset-start] = indicator } } } return frame } // Keep long Unicode lines inside the helper frame limit without losing the // cursor. This is a presentation window; the document keeps the full line. func ocr_braille_frame(position ocr.Position) braille.Frame { if len(position.Line) <= 10000 { return braille.Frame{Text: position.Line, Cursor: position.Cursor} } const windowRunes = 2000 // At most 8000 UTF-8 bytes, below the helper limit. length := utf8.RuneCountInString(position.Line) start := max(0, min(position.Cursor-windowRunes/2, length-windowRunes)) end := min(length, start+windowRunes) startByte, endByte, offset := 0, len(position.Line), 0 for index := range position.Line { if offset == start { startByte = index } if offset == end { endByte = index break } offset++ } return braille.Frame{Text: position.Line[startByte:endByte], Cursor: position.Cursor - start} } func presentationOCRText(text string, unit ocr.Unit) string { if text == "" { return "blank" } if unit == ocr.Character || unit == ocr.LineStart || unit == ocr.LineEnd { switch text { case " ": return "space" case "\n": return "new line" case "\t": return "tab" } } return text } // Observe runs before native event presentation. Focus/owner changes retire // synchronously, including changes within one X11 client. OCR never invents an // AT-SPI object; duplicate source updates cannot overwrite its virtual cursor. func (session *ocrSession) Observe(event atspi.Event) bool { if session == nil { return false } session.mu.Lock() defer session.mu.Unlock() if session.activating { if (isFocusGainEvent(event) && event.Source.ID().BusName == session.job.focus.BusName && event.Source.ID() != session.job.focus) || (event.Type == atspi.DefunctStateEvent && (event.Source.ID() == session.job.focus || event.Source.ID() == session.source.Object)) { session.activationInterrupted = true } return false } if session.cancel != nil && session.foreground.Foreground() != session.job.foreground { session.retire_locked() } if isFocusGainEvent(event) && !event.ForegroundMismatch && !eventBypassesXTermHandoff(event) && (session.foreground == nil || event.Foreground.Epoch == 0 || event.Foreground.Epoch == session.foreground.Foreground().Epoch) { if event.Type == atspi.WindowActivatedEvent && session.window.Valid() && eventWindow(event) == session.window { return session.cursor != nil } session.window = eventWindow(event) if session.focus != event.Source.ID() { session.retire_locked() } session.focus = event.Source.ID() if session.foreground != nil { session.focusForeground = session.foreground.Foreground().Client.Window } } if event.Type == atspi.WindowDeactivatedEvent || event.Type == atspi.WindowDestroyedEvent || event.Type == atspi.DefunctStateEvent { if session.foreground != nil && transientForegroundDeactivation(event, session.foreground.Foreground()) { return false } sourceLost := event.Source.ID() == session.focus || session.cancel != nil && (event.Source.ID() == session.source.Object || event.Source.ID() == session.job.window) windowLost := event.Type != atspi.DefunctStateEvent && session.cancel != nil && eventWindow(event) == session.job.window if sourceLost || windowLost { session.retire_locked() session.focus = atspi.ObjectID{} } } return session.cursor != nil && !eventBypassesXTermHandoff(event) && (isFocusGainEvent(event) || event.Type == atspi.CaretEvent || isTextChangeEvent(event.Type) || event.Type == atspi.AccessibleNameEvent || event.Type == atspi.TextSelectionChangedEvent) } func (session *ocrSession) ForegroundChanged() { if session == nil { return } session.mu.Lock() defer session.mu.Unlock() if session.activating { return } if session.cancel != nil && session.foreground.Foreground() != session.job.foreground { session.retire_locked() } } func (session *ocrSession) settings() config.Settings { if session.live != nil { return session.live.Load() } return config.DefaultSettings() } func (session *ocrSession) Reviewing() bool { if session == nil { return false } session.mu.Lock() defer session.mu.Unlock() return session.cancel != nil && session.cursor != nil } // RouteBraille accepts only a current OCR frame's generation and moves the // virtual cursor. It never forwards a route to AT-SPI or application focus. func (session *ocrSession) RouteBraille(ctx context.Context, generation uint64, offset int) { if session == nil || ctx.Err() != nil { return } session.mu.Lock() defer session.mu.Unlock() if generation != session.generation || session.cursor == nil || !session.current_locked(session.job) || ctx.Err() != nil { return } position, ok := session.cursor.Set(offset) if !ok { return } session.selectionAnchor = -1 session.logger.Debug("OCR braille route", "command", "braille-route", "generation", generation, "focus_path", session.focus.Path, "source_path", session.source.Object.Path, "offset", offset) session.present_locked(position, ocr.Character) }