Better handle punctuation inside words like email addresses, urls, file paths, etc.

This commit is contained in:
Storm Dragon committed 2026-10-28 20:52:18 -04:00
1 parent 7c53cb39d4
commit 7df1fa3ecb
4 files changed
+101 -7

No files matched your search

+33 -5
View File
@@ -13,6 +13,7 @@ import (
"os"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"
"git.stormux.org/storm/fenrir/internal/i18n"
@@ -156,14 +157,37 @@ func (p *Processor) Character(char rune) string {
return presentation.Character(char)
}
// Text follows Python's levels and pause handling, retaining unspoken sentence
// punctuation and replacing other unspoken symbols with spaces. Numbers and
// contractions therefore retain their original punctuation at lower levels.
// Text retains Python's levels and pause handling. At some and most, symbols
// embedded between letters or numbers are also named, except apostrophes in
// contractions and single numeric separators. Sentence punctuation still follows
// the selected level.
func (p *Processor) Text(text, level string) string {
selected := p.levels[level]
var result strings.Builder
for _, char := range text {
if selected[char] {
chars := []rune(text)
embeddedEnd := 0
embedded := false
for index, char := range chars {
if index >= embeddedEnd {
embeddedEnd = index + 1
embedded = false
if (level == "some" || level == "most") && (unicode.IsPunct(char) || unicode.IsSymbol(char)) {
for embeddedEnd < len(chars) && (unicode.IsPunct(chars[embeddedEnd]) || unicode.IsSymbol(chars[embeddedEnd])) {
embeddedEnd++
}
embedded = index > 0 && embeddedEnd < len(chars) &&
is_word_rune(chars[index-1]) && is_word_rune(chars[embeddedEnd])
if embedded && embeddedEnd == index+1 {
before, after := chars[index-1], chars[embeddedEnd]
if char == '\'' && unicode.IsLetter(before) && unicode.IsLetter(after) ||
strings.ContainsRune(".,:", char) && unicode.IsDigit(before) && unicode.IsDigit(after) {
embedded = false
}
}
}
}
_, known := p.symbols[char]
if selected[char] || embedded && known {
result.WriteByte(' ')
result.WriteString(p.Character(char))
if strings.ContainsRune("',.;:?!", char) {
@@ -180,3 +204,7 @@ func (p *Processor) Text(text, level string) string {
}
return result.String()
}
func is_word_rune(char rune) bool {
return unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsMark(char)
}