storm/fenrir
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug where punctuation was skipped when occuring inside numbers. Also removed the end of sentence pause marker for punctuation inside words/numbers.

Browse Source
This commit is contained in:
Storm Dragon committed 2026-10-30 11:54:15 -04:00
1 parent 22f6df6af9
commit 36736725ea
4 files changed
+85 -26
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/punctuation/profile_test.go
+45 -12
View File
@@ -40,7 +40,7 @@ func TestTextLevelsPreservePausesAndDoNotReprocessNames(t *testing.T) {
for _, testCase := range []struct{ level, input, want string }{
{"none", "don't 3.14 [X]mute a_b", "don't 3.14 X mute a b"},
{"some", "a/b#c!", "a slash b hash c!"},
{"most", "a,b!", "a comma, b bang! "},
{"most", "a,b!", "a comma b bang! "},
{"all", "?", " question?? "},
{"all", "a b", "a space b"},
{"some", "é→Ω", "é→Ω"},
@@ -63,17 +63,17 @@ func TestEmbeddedPunctuation(t *testing.T) {
t.Fatal(err)
}
for _, trial := range []struct{ input, want string }{
{"https://stormux.org", "https colon: slash slash stormux dot. org"},
{"Visit https://example.invalid/path?q=a&n=2.", "Visit https colon: slash slash example dot. invalid slash path question?? q equals a and n equals 2."},
{"user@example.invalid", "user at example dot. invalid"},
{"file_name.txt", "file line name dot. txt"},
{"你好.世界 café:menu", "你好 dot. 世界 café colon: menu"},
{"cafe\u0301.txt", "cafe\u0301 dot. txt"},
{"https://stormux.org", "https colon slash slash stormux dot org"},
{"Visit https://example.invalid/path?q=a&n=2.", "Visit https colon slash slash example dot invalid slash path question? q equals a and n equals 2."},
{"user@example.invalid", "user at example dot invalid"},
{"file_name.txt", "file line name dot txt"},
{"你好.世界 café:menu", "你好 dot 世界 café colon menu"},
{"cafe\u0301.txt", "cafe\u0301 dot txt"},
{"don't can't", "don't can't"},
{"3.14 1,000 12:30", "3.14 1,000 12:30"},
{"Hello, world! End. Next: word?", "Hello, world! End. Next: word?"},
{"(word) [word]", "word word"},
{"a...b", "a dot. dot. dot. b"},
{"a...b", "a dot dot dot b"},
} {
if got := strings.Join(strings.Fields(processor.Text(trial.input, "some")), " "); got != trial.want {
t.Errorf("%q: %q, want %q", trial.input, got, trial.want)
@@ -81,9 +81,9 @@ func TestEmbeddedPunctuation(t *testing.T) {
}
for _, trial := range []struct{ level, want string }{
{"none", "https: stormux.org"},
{"some", "https colon: slash slash stormux dot. org"},
{"most", "https colon: slash slash stormux dot. org"},
{"all", "https colon: slash slash stormux dot. org"},
{"some", "https colon slash slash stormux dot org"},
{"most", "https colon slash slash stormux dot org"},
{"all", "https colon slash slash stormux dot org"},
} {
if got := processor.Text("https://stormux.org", trial.level); got != trial.want {
t.Errorf("%s: %q, want %q", trial.level, got, trial.want)
@@ -93,11 +93,44 @@ func TestEmbeddedPunctuation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if got := french.Text("example.invalid", "some"); got != "example point. invalid" {
if got := french.Text("example.invalid", "some"); got != "example point invalid" {
t.Fatal("embedded punctuation lost profile name", got)
}
}
func TestDottedNumericSequences(t *testing.T) {
processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
if err != nil {
t.Fatal(err)
}
for _, trial := range []struct{ input, want string }{
{"2026.10.31", "2026 dot 10 dot 31"},
{"(2026.10.31).", "2026 dot 10 dot 31 ."},
{"v1.2.3 127.0.0.1", "v1 dot 2 dot 3 127 dot 0 dot 0 dot 1"},
{"２０２６.１０.３１", "２０２６ dot １０ dot ３１"},
{"3.14 2026.10.31 2.5", "3.14 2026 dot 10 dot 31 2.5"},
{"3.14. 2.5", "3.14. 2.5"},
{"1,234.56 1.234,56 1,000,000 12:30:45", "1,234.56 1.234,56 1,000,000 12:30:45"},
} {
got := strings.Join(strings.Fields(processor.Text(trial.input, "some")), " ")
if got != trial.want {
t.Errorf("%q: got %q, want %q", trial.input, got, trial.want)
}
}
for _, trial := range []struct{ level, want string }{
{"some", "2026 dot 10 dot 31."},
{"most", "2026 dot 10 dot 31 dot. "},
{"all", "2026 dot 10 dot 31 dot. "},
} {
if got := processor.Text("2026.10.31.", trial.level); got != trial.want {
t.Errorf("%s: incorrect embedded or sentence pause: %q", trial.level, got)
}
}
if got := processor.Text("2026.10.31", "none"); got != "2026.10.31" {
t.Fatal("none level changed", got)
}
}
func TestNumericPercentSpeech(t *testing.T) {
processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
if err != nil {