Better handle punctuation inside words like email addresses, urls, file paths, etc.
This commit is contained in:
1 parent
7c53cb39d4
commit
7df1fa3ecb
4 files changed
+101
-7
No files matched your search
@@ -13,6 +13,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.stormux.org/storm/fenrir/internal/i18n"
|
||||
@@ -156,14 +157,37 @@ func (p *Processor) Character(char rune) string {
|
||||
return presentation.Character(char)
|
||||
}
|
||||
|
||||
// Text follows Python's levels and pause handling, retaining unspoken sentence
|
||||
// punctuation and replacing other unspoken symbols with spaces. Numbers and
|
||||
// contractions therefore retain their original punctuation at lower levels.
|
||||
// Text retains Python's levels and pause handling. At some and most, symbols
|
||||
// embedded between letters or numbers are also named, except apostrophes in
|
||||
// contractions and single numeric separators. Sentence punctuation still follows
|
||||
// the selected level.
|
||||
func (p *Processor) Text(text, level string) string {
|
||||
selected := p.levels[level]
|
||||
var result strings.Builder
|
||||
for _, char := range text {
|
||||
if selected[char] {
|
||||
chars := []rune(text)
|
||||
embeddedEnd := 0
|
||||
embedded := false
|
||||
for index, char := range chars {
|
||||
if index >= embeddedEnd {
|
||||
embeddedEnd = index + 1
|
||||
embedded = false
|
||||
if (level == "some" || level == "most") && (unicode.IsPunct(char) || unicode.IsSymbol(char)) {
|
||||
for embeddedEnd < len(chars) && (unicode.IsPunct(chars[embeddedEnd]) || unicode.IsSymbol(chars[embeddedEnd])) {
|
||||
embeddedEnd++
|
||||
}
|
||||
embedded = index > 0 && embeddedEnd < len(chars) &&
|
||||
is_word_rune(chars[index-1]) && is_word_rune(chars[embeddedEnd])
|
||||
if embedded && embeddedEnd == index+1 {
|
||||
before, after := chars[index-1], chars[embeddedEnd]
|
||||
if char == '\'' && unicode.IsLetter(before) && unicode.IsLetter(after) ||
|
||||
strings.ContainsRune(".,:", char) && unicode.IsDigit(before) && unicode.IsDigit(after) {
|
||||
embedded = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_, known := p.symbols[char]
|
||||
if selected[char] || embedded && known {
|
||||
result.WriteByte(' ')
|
||||
result.WriteString(p.Character(char))
|
||||
if strings.ContainsRune("',.;:?!", char) {
|
||||
@@ -180,3 +204,7 @@ func (p *Processor) Text(text, level string) string {
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
|
||||
func is_word_rune(char rune) bool {
|
||||
return unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsMark(char)
|
||||
}
|
||||
Reference in new issue
Block a user