Better handle punctuation inside words like email addresses, urls, file paths, etc.

This commit is contained in:
Storm Dragon committed 2026-10-28 20:52:18 -04:00
1 parent 7c53cb39d4
commit 7df1fa3ecb
4 files changed
+101 -7

No files matched your search

+41
View File
@@ -57,6 +57,47 @@ func TestTextLevelsPreservePausesAndDoNotReprocessNames(t *testing.T) {
}
}
func TestEmbeddedPunctuation(t *testing.T) {
processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
if err != nil {
t.Fatal(err)
}
for _, trial := range []struct{ input, want string }{
{"https://stormux.org", "https colon: slash slash stormux dot. org"},
{"Visit https://example.invalid/path?q=a&n=2.", "Visit https colon: slash slash example dot. invalid slash path question?? q equals a and n equals 2."},
{"user@example.invalid", "user at example dot. invalid"},
{"file_name.txt", "file line name dot. txt"},
{"你好.世界 café:menu", "你好 dot. 世界 café colon: menu"},
{"cafe\u0301.txt", "cafe\u0301 dot. txt"},
{"don't can't", "don't can't"},
{"3.14 1,000 12:30", "3.14 1,000 12:30"},
{"Hello, world! End. Next: word?", "Hello, world! End. Next: word?"},
{"(word) [word]", "word word"},
{"a...b", "a dot. dot. dot. b"},
} {
if got := strings.Join(strings.Fields(processor.Text(trial.input, "some")), " "); got != trial.want {
t.Errorf("%q: %q, want %q", trial.input, got, trial.want)
}
}
for _, trial := range []struct{ level, want string }{
{"none", "https: stormux.org"},
{"some", "https colon: slash slash stormux dot. org"},
{"most", "https colon: slash slash stormux dot. org"},
{"all", "https colon: slash slash stormux dot. org"},
} {
if got := processor.Text("https://stormux.org", trial.level); got != trial.want {
t.Errorf("%s: %q, want %q", trial.level, got, trial.want)
}
}
french, _, err := Load([]string{"../../config/punctuation/fr.toml"})
if err != nil {
t.Fatal(err)
}
if got := french.Text("example.invalid", "some"); got != "example point. invalid" {
t.Fatal("embedded punctuation lost profile name", got)
}
}
func TestCustomNamesAndInvalidProfiles(t *testing.T) {
data, err := os.ReadFile("../../config/punctuation/default.toml")
if err != nil {