// Copyright (c) 2026 Storm Dragon // Authors: Storm Dragon, Angus Kola // SPDX-License-Identifier: GPL-3.0-or-later package ocr import ( "errors" "image" "strings" "testing" "unicode/utf8" ) func TestParseTSVHierarchyUnicodeAndEngineOrder(t *testing.T) { data := tsvHeader + "\n" + "1\t1\t0\t0\t0\t0\t0\t0\t200\t100\t-1\t\n" + "2\t1\t1\t0\t0\t0\t0\t0\t200\t100\t-1\t\n" + "3\t1\t1\t1\t0\t0\t0\t0\t200\t100\t-1\t\n" + "4\t1\t1\t1\t1\t0\t0\t0\t200\t100\t-1\t\n" + "5\t1\t1\t1\t1\t1\t100\t50\t30\t20\t0\té\n" + "5\t1\t1\t1\t1\t2\t10\t50\t30\t20\t-1\t中文\n" + "5\t1\t1\t2\t1\t1\t10\t10\t30\t20\t95.25\te\u0301\n" + "5\t2\t1\t1\t1\t1\t10\t10\t30\t20\t85\tNext" result, err := ParseTSV(strings.NewReader(data), 200, 100) if err != nil { t.Fatal(err) } if result.Text != "é 中文\ne\u0301\nNext" || len(result.Lines) != 3 || len(result.Words) != 4 { t.Fatalf("wrong assembly: %+v", result) } if result.Words[1].Span != (Span{2, 4}) || result.Words[2].Span != (Span{5, 7}) || result.Words[1].Confidence != -1 || result.Words[0].Confidence != 0 || result.Words[0].Bounds != image.Rect(100, 50, 130, 70) { t.Fatalf("lost Unicode offsets, low confidence or coordinates: %+v", result.Words) } if result.Lines[1].ID != (LineID{1, 1, 2, 1}) || result.Lines[1].Span != (Span{5, 7}) || result.Lines[1].WordStart != 2 || result.Lines[1].WordEnd != 3 { t.Fatalf("lost paragraph/line ownership: %+v", result.Lines) } } func TestParseTSVEmptyAndQuotedWords(t *testing.T) { for _, data := range []string{tsvHeader + "\n", tsvHeader + "\r\n1\t1\t0\t0\t0\t0\t0\t0\t100\t100\t-1\t\r\n"} { result, err := ParseTSV(strings.NewReader(data), 100, 100) if err != nil || result.Text != "" || len(result.Words) != 0 { t.Fatalf("empty result: %+v, %v", result, err) } } data := tsvHeader + "\n5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t99\t\"word\"\n" result, err := ParseTSV(strings.NewReader(data), 100, 100) if err != nil || result.Text != `"word"` { t.Fatalf("TSV word treated as CSV quoting: %+v, %v", result, err) } } func TestParseTSVRejectsMalformedOutputAtomically(t *testing.T) { row := "5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t90\tPrivate fixture text" for _, bad := range []string{ "missing fields", strings.Replace(row, "5\t", "6\t", 1), strings.Replace(row, "5\t1", "5\t0", 1), strings.Replace(row, "0\t0\t10\t10", "-1\t0\t10\t10", 1), strings.Replace(row, "0\t0\t10\t10", "95\t0\t10\t10", 1), strings.Replace(row, "0\t0\t10\t10", "0\t0\t0\t10", 1), strings.Replace(row, "0\t0\t10\t10", "0\t0\t2147483647\t10", 1), strings.Replace(row, "90\t", "NaN\t", 1), strings.Replace(row, "90\t", "+Inf\t", 1), strings.Replace(row, "90\t", "101\t", 1), strings.Replace(row, "90\t", "-2\t", 1), row + "\x00", row + "\xff", row + "\textra field", row + " ", } { result, err := ParseTSV(strings.NewReader(tsvHeader+"\n"+row+"\n"+bad+"\n"), 100, 100) if !errors.Is(err, ErrInvalidOutput) || result.Text != "" || len(result.Words) != 0 { t.Fatalf("accepted malformed row or partial document: %+v, %v", result, err) } if strings.Contains(err.Error(), "Private") { t.Fatal("recognized text leaked into error") } } if _, err := ParseTSV(strings.NewReader("not a TSV header"), 100, 100); !errors.Is(err, ErrInvalidOutput) { t.Fatal("invalid header accepted") } if _, err := ParseTSV(strings.NewReader(tsvHeader), MaxPixels, 2); !errors.Is(err, ErrInvalidImage) { t.Fatal("pixel limit ignored") } } func TestParseTSVLimitsAndRevisitedLine(t *testing.T) { if _, err := ParseTSV(strings.NewReader(tsvHeader+"\n"+strings.Repeat("x", 64<<10)), 100, 100); !errors.Is(err, ErrLimit) { t.Fatalf("unbounded row: %v", err) } row := "5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t90\tword\n" secondLine := strings.Replace(row, "1\t1\t1\t1\t1\t", "1\t1\t1\t2\t1\t", 1) if _, err := ParseTSV(strings.NewReader(tsvHeader+"\n"+row+secondLine+row), 100, 100); !errors.Is(err, ErrInvalidOutput) { t.Fatalf("revisited line accepted: %v", err) } largeWord := "5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t90\t" + strings.Repeat("x", 60<<10) + "\n" if _, err := ParseTSV(strings.NewReader(tsvHeader+"\n"+strings.Repeat(largeWord, 20)), 100, 100); !errors.Is(err, ErrLimit) { t.Fatalf("text limit ignored: %v", err) } if _, err := ParseTSV(strings.NewReader(tsvHeader+"\n"+strings.Repeat(row, maxWords+1)), 100, 100); !errors.Is(err, ErrLimit) { t.Fatalf("word limit ignored: %v", err) } } func FuzzParseTSV(f *testing.F) { f.Add(tsvHeader + "\n5\t1\t1\t1\t1\t1\t1\t1\t10\t10\t90\t中文\n") f.Add("") f.Fuzz(func(t *testing.T, data string) { result, err := ParseTSV(strings.NewReader(data), 200, 100) if err != nil { if result.Text != "" || len(result.Words) != 0 { t.Fatal("partial result escaped a parser failure") } return } if !utf8.ValidString(result.Text) || len(result.Text) > MaxTextBytes { t.Fatal("invalid document text") } lastEnd := 0 for _, word := range result.Words { start, startOK := result.ByteOffset(word.Start) end, endOK := result.ByteOffset(word.End) if !startOK || !endOK || word.Start < lastEnd || word.End <= word.Start || result.Text[start:end] != word.Text || !valid_word_bounds(word.Bounds, result.Width, result.Height) { t.Fatal("invalid word map") } lastEnd = word.End } }) }