Better handle punctuation inside words like email addresses, urls, file paths, etc.
This commit is contained in:
1 parent
7c53cb39d4
commit
7df1fa3ecb
4 files changed
+101
-7
No files matched your search
@@ -57,6 +57,47 @@ func TestTextLevelsPreservePausesAndDoNotReprocessNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedPunctuation(t *testing.T) {
|
||||
processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, trial := range []struct{ input, want string }{
|
||||
{"https://stormux.org", "https colon: slash slash stormux dot. org"},
|
||||
{"Visit https://example.invalid/path?q=a&n=2.", "Visit https colon: slash slash example dot. invalid slash path question?? q equals a and n equals 2."},
|
||||
{"user@example.invalid", "user at example dot. invalid"},
|
||||
{"file_name.txt", "file line name dot. txt"},
|
||||
{"你好.世界 café:menu", "你好 dot. 世界 café colon: menu"},
|
||||
{"cafe\u0301.txt", "cafe\u0301 dot. txt"},
|
||||
{"don't can't", "don't can't"},
|
||||
{"3.14 1,000 12:30", "3.14 1,000 12:30"},
|
||||
{"Hello, world! End. Next: word?", "Hello, world! End. Next: word?"},
|
||||
{"(word) [word]", "word word"},
|
||||
{"a...b", "a dot. dot. dot. b"},
|
||||
} {
|
||||
if got := strings.Join(strings.Fields(processor.Text(trial.input, "some")), " "); got != trial.want {
|
||||
t.Errorf("%q: %q, want %q", trial.input, got, trial.want)
|
||||
}
|
||||
}
|
||||
for _, trial := range []struct{ level, want string }{
|
||||
{"none", "https: stormux.org"},
|
||||
{"some", "https colon: slash slash stormux dot. org"},
|
||||
{"most", "https colon: slash slash stormux dot. org"},
|
||||
{"all", "https colon: slash slash stormux dot. org"},
|
||||
} {
|
||||
if got := processor.Text("https://stormux.org", trial.level); got != trial.want {
|
||||
t.Errorf("%s: %q, want %q", trial.level, got, trial.want)
|
||||
}
|
||||
}
|
||||
french, _, err := Load([]string{"../../config/punctuation/fr.toml"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := french.Text("example.invalid", "some"); got != "example point. invalid" {
|
||||
t.Fatal("embedded punctuation lost profile name", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomNamesAndInvalidProfiles(t *testing.T) {
|
||||
data, err := os.ReadFile("../../config/punctuation/default.toml")
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user