Cthulhu is no longer a fork. Now it is a screen reader written in Golang.
This commit is contained in:
Storm Dragon committed
1 parent
a998fe67de
commit
069fc674dc
708 files changed
+112482
-532
No files matched your search
@@ -0,0 +1,242 @@
|
||||
// Copyright (c) 2026 Storm Dragon
|
||||
// Authors: Storm Dragon, Angus Kola
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.stormux.org/storm/cthulhu/internal/atspi"
|
||||
"git.stormux.org/storm/cthulhu/internal/ocr"
|
||||
)
|
||||
|
||||
type ocrActivateRequest struct {
|
||||
job ocrJob
|
||||
word ocr.Word
|
||||
capture ocr.Capture
|
||||
}
|
||||
|
||||
type ocrDialogConfirmer struct{ executable string }
|
||||
|
||||
func (helper ocrDialogConfirmer) Confirm(ctx context.Context, word string) (bool, error) {
|
||||
if len(word) == 0 || len(word) > 4096 || strings.ContainsRune(word, 0) {
|
||||
return false, errors.New("invalid OCR word")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(ctx, helper.executable, "--ocr-confirm")
|
||||
command.Stdin = strings.NewReader(word)
|
||||
var output bytes.Buffer
|
||||
command.Stdout = &output
|
||||
if err := command.Run(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if output.String() != "yes\n" && output.String() != "cancel\n" {
|
||||
return false, errors.New("invalid OCR confirmation")
|
||||
}
|
||||
return output.String() == "yes\n", nil
|
||||
}
|
||||
|
||||
func (session *ocrSession) activate_locked() {
|
||||
if session.cursor == nil || session.refreshing || session.refreshFailed || session.activating || session.clicker == nil || session.activateRequests == nil ||
|
||||
!session.current_locked(session.job) || session.job.kind == ocr.ClipboardSource {
|
||||
session.status_locked("OCR activation unavailable; refresh the screen result")
|
||||
return
|
||||
}
|
||||
word, ok := session.cursor.CurrentWord()
|
||||
if !ok {
|
||||
session.status_locked("Move to a recognized word to activate it")
|
||||
return
|
||||
}
|
||||
if _, ok = session.pixels.Transform.ScreenRectangle(word.Bounds); !ok {
|
||||
session.status_locked("OCR word has no screen position")
|
||||
return
|
||||
}
|
||||
if session.autoTimer != nil {
|
||||
session.autoTimer.Stop()
|
||||
session.autoTimer = nil
|
||||
}
|
||||
session.activating = true
|
||||
session.activationInterrupted = false
|
||||
job := session.job
|
||||
job.source = session.source
|
||||
session.activateRequests <- ocrActivateRequest{job: job, word: word, capture: session.pixels}
|
||||
}
|
||||
|
||||
func (session *ocrSession) activation_current(request ocrActivateRequest) bool {
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
return session.activating && !session.activationInterrupted && session.current_locked(request.job)
|
||||
}
|
||||
|
||||
func (session *ocrSession) activate(request ocrActivateRequest) {
|
||||
if !session.activation_current(request) {
|
||||
session.finish_activation(request, true, "OCR source changed")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(request.job.ctx, 35*time.Second)
|
||||
defer cancel()
|
||||
if err := session.keys.SetOCRReview(ctx, false); err != nil {
|
||||
session.mu.Lock()
|
||||
current := session.activating && session.generation == request.job.generation && session.cancel != nil
|
||||
if current {
|
||||
session.retire_locked()
|
||||
}
|
||||
session.mu.Unlock()
|
||||
if current && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
|
||||
select {
|
||||
case session.failures <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if current {
|
||||
session.release_input()
|
||||
}
|
||||
return
|
||||
}
|
||||
if session.settings().OCR.ConfirmClick {
|
||||
if session.confirmer == nil {
|
||||
session.finish_activation(request, false, "OCR confirmation unavailable")
|
||||
return
|
||||
}
|
||||
confirmed, err := session.confirmer.Confirm(ctx, request.word.Text)
|
||||
session.wait_activation_source(ctx, request)
|
||||
if err != nil || !confirmed {
|
||||
session.finish_activation(request, false, "OCR activation cancelled")
|
||||
return
|
||||
}
|
||||
}
|
||||
if !session.activation_current(request) {
|
||||
session.finish_activation(request, true, "OCR source changed")
|
||||
return
|
||||
}
|
||||
if err := session.validate_activation(ctx, request); err != nil {
|
||||
session.finish_activation(request, true, "OCR source changed; refresh before activation")
|
||||
return
|
||||
}
|
||||
rectangle, ok := request.capture.Transform.ScreenRectangle(request.word.Bounds)
|
||||
if !ok {
|
||||
session.finish_activation(request, true, "OCR word has no screen position")
|
||||
return
|
||||
}
|
||||
point := image.Pt(rectangle.Min.X+(rectangle.Dx()-1)/2, rectangle.Min.Y+(rectangle.Dy()-1)/2)
|
||||
session.mu.Lock()
|
||||
if !session.activating || session.activationInterrupted || !session.current_locked(request.job) {
|
||||
session.mu.Unlock()
|
||||
session.finish_activation(request, true, "OCR source changed")
|
||||
return
|
||||
}
|
||||
session.retire_locked()
|
||||
session.mu.Unlock()
|
||||
clickCtx, clickCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer clickCancel()
|
||||
if err := session.clicker.ClickOCR(clickCtx, request.job.foreground.Client.Window, request.capture.RootBounds, point); err != nil {
|
||||
session.mu.Lock()
|
||||
session.status_locked("OCR click unavailable")
|
||||
session.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (session *ocrSession) wait_activation_source(ctx context.Context, request ocrActivateRequest) {
|
||||
deadline := time.NewTimer(time.Second)
|
||||
defer deadline.Stop()
|
||||
ticker := time.NewTicker(20 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for !session.activation_current(request) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-deadline.C:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (session *ocrSession) finish_activation(request ocrActivateRequest, stale bool, message string) {
|
||||
session.mu.Lock()
|
||||
if !session.activating || session.generation != request.job.generation || request.job.ctx.Err() != nil {
|
||||
session.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if session.activationInterrupted || !session.current_locked(request.job) {
|
||||
session.retire_locked()
|
||||
session.status_locked("OCR source changed")
|
||||
session.mu.Unlock()
|
||||
return
|
||||
}
|
||||
session.mu.Unlock()
|
||||
// Re-enter only in the original client. Native entry waits for complete
|
||||
// dialog chord release and refuses changed foreground ownership.
|
||||
ctx, cancel := context.WithTimeout(request.job.ctx, 2*time.Second)
|
||||
err := session.keys.SetOCRContext(ctx, request.job.foreground.Client.Window, false)
|
||||
cancel()
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
if !session.activating || session.activationInterrupted || !session.current_locked(request.job) {
|
||||
session.retire_locked()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
session.retire_locked()
|
||||
session.status_locked("OCR review unavailable")
|
||||
return
|
||||
}
|
||||
session.activating = false
|
||||
if stale {
|
||||
session.refreshFailed = true
|
||||
} else {
|
||||
session.schedule_refresh_locked()
|
||||
}
|
||||
session.status_locked(message)
|
||||
}
|
||||
|
||||
func (session *ocrSession) validate_activation(ctx context.Context, request ocrActivateRequest) error {
|
||||
if request.capture.Pixels == nil {
|
||||
return ocr.ErrSourceChanged
|
||||
}
|
||||
if request.job.kind == ocr.ObjectSource {
|
||||
var zone *atspi.Rectangle
|
||||
if request.job.source.HasZone {
|
||||
zone = &request.job.source.Zone
|
||||
}
|
||||
geometry, err := session.geometry.CaptureOCRGeometry(ctx, request.job.source.Object, zone)
|
||||
if err != nil || geometry.Foreground != request.job.foreground || geometry.ScreenBounds != request.capture.Request.Bounds {
|
||||
return ocr.ErrSourceChanged
|
||||
}
|
||||
}
|
||||
fresh, err := session.capture.Capture(ctx, request.capture.Request)
|
||||
if err != nil || fresh.SourceWindow != request.capture.SourceWindow || fresh.RootBounds != request.capture.RootBounds ||
|
||||
fresh.SourceBounds != request.capture.SourceBounds || fresh.ScreenBounds != request.capture.ScreenBounds ||
|
||||
!equalOCRPixels(ctx, fresh.Pixels, request.capture.Pixels) {
|
||||
return ocr.ErrSourceChanged
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func equalOCRPixels(ctx context.Context, first, second image.Image) bool {
|
||||
if first == nil || second == nil || first.Bounds() != second.Bounds() {
|
||||
return false
|
||||
}
|
||||
bounds := first.Bounds()
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
if y%32 == 0 && ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
r1, g1, b1, a1 := first.At(x, y).RGBA()
|
||||
r2, g2, b2, a2 := second.At(x, y).RGBA()
|
||||
if r1 != r2 || g1 != g2 || b1 != b2 || a1 != a2 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
Reference in new issue
Block a user