After commit c018f5fd29 Fenrir is being switched over to go. If you do not want to be part of the testing process, please switch to master.

internal/punctuation/text_processing_test.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2026 Storm Dragon
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package punctuation
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.stormux.org/storm/fenrir/internal/i18n"
+)
+
+func TestTextSubstitutionsAndBoundaries(t *testing.T) {
+	processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	options := TextOptions{Emoticons: true, RepeatedSymbols: true, RepeatedSymbolsMin: 4}
+	for _, trial := range []struct{ text, want string }{
+		{":)", "smile"},
+		{":) :) :)", "smile smile smile"},
+		{":) :(", "smile frown"},
+		{"hello :) there", "hello smile there"},
+		{"你好 :) 世界", "你好 smile 世界"},
+		{"#####", "5 number"},
+		{"#### ####", "4 number 4 number"},
+		{"##### title #####", "5 number title 5 number"},
+		{"////", "4 slash"},
+		{"Oo", "WTF?"},
+	} {
+		if got := processor.Process_text(trial.text, "some", options); got != trial.want {
+			t.Errorf("%q: %q, want %q", trial.text, got, trial.want)
+		}
+	}
+	for _, text := range []string{"hello :)!", "path/to:)file", "https://example.invalid/:)", "https://example.invalid/:-/", "(o_O)", "###", "##--##", "a####b", "../....", "https://example.invalid/####", "10000", "aaaa", "🙂🙂🙂🙂", ""} {
+		for _, level := range Levels() {
+			if got, want := processor.Process_text(text, level, options), processor.Text(text, level); got != want {
+				t.Errorf("%q, %s changed ordinary processing: %q, want %q", text, level, got, want)
+			}
+		}
+	}
+	if got := processor.Process_text("\t:)\u00a0:) \n", "none", options); got != "\tsmile\u00a0smile \n" {
+		t.Fatal("whitespace changed", got)
+	}
+	if got := processor.Process_text("##### ####", "all", options); got != "5 number space 4 number" {
+		t.Fatal("generated names were processed again", got)
+	}
+	if got := processor.Process_text("Oo", "all", options); got != "WTF?" {
+		t.Fatal("punctuation in emoticon name processed again", got)
+	}
+}
+
+func TestSubstitutionOptionsAndProfileNames(t *testing.T) {
+	processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	text := ":) #####"
+	for _, level := range Levels() {
+		if got := processor.Process_text(text, level, TextOptions{}); got != processor.Text(text, level) {
+			t.Fatal("disabled substitutions changed text", level, got)
+		}
+	}
+	options := TextOptions{RepeatedSymbols: true, RepeatedSymbolsMin: 5}
+	if got := processor.Process_text("####", "some", options); got != processor.Text("####", "some") {
+		t.Fatal("minimum ignored", got)
+	}
+	if got := processor.Process_text("#####", "none", options); got != processor.Text("#####", "none") {
+		t.Fatal("summarized unselected symbol", got)
+	}
+	for _, minimum := range []int{0, 1} {
+		options.RepeatedSymbolsMin = minimum
+		if got := processor.Process_text("#####", "some", options); got != processor.Text("#####", "some") {
+			t.Fatal("invalid options summarized text", got)
+		}
+	}
+	options.RepeatedSymbolsMin = 4
+	data, err := os.ReadFile("../../config/punctuation/default.toml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data = []byte(strings.Replace(string(data), `"/" = "slash"`, `"/" = "path/separator"`, 1))
+	custom, err := Decode(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := custom.Process_text("////", "some", options); got != "4 path/separator" {
+		t.Fatal("custom name changed or processed twice", got)
+	}
+	french, _, err := Load([]string{"../../config/punctuation/fr.toml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := french.Process_text("#####", "most", options); got != "5 dièse" {
+		t.Fatal("localized profile name lost", got)
+	}
+	if got := processor.Character('#'); got != "hash" {
+		t.Fatal("summary changed character naming", got)
+	}
+}
+
+func TestTextSubstitutionTranslationsAndFieldOrder(t *testing.T) {
+	root := t.TempDir()
+	directory := filepath.Join(root, "xx", "LC_MESSAGES")
+	if err := os.MkdirAll(directory, 0700); err != nil {
+		t.Fatal(err)
+	}
+	catalog := `msgid ""
+msgstr ""
+"Content-Type: text/plain; charset=UTF-8\n"
+
+msgid "smile"
+msgstr "translated-smile"
+
+msgid "number"
+msgstr "translated-number"
+
+msgid "{0} {1}"
+msgstr "{1}: {0}"
+`
+	if err := os.WriteFile(filepath.Join(directory, "fenrir.po"), []byte(catalog), 0600); err != nil {
+		t.Fatal(err)
+	}
+	previous := i18n.Use(i18n.New(root, "xx"))
+	t.Cleanup(func() { i18n.Use(previous) })
+	processor, _, err := Load([]string{"../../config/punctuation/default.toml"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	options := TextOptions{Emoticons: true, RepeatedSymbols: true, RepeatedSymbolsMin: 4}
+	if got := processor.Process_text(":) #####", "some", options); got != "translated-smile translated-number: 5" {
+		t.Fatal("translation or positional order lost", got)
+	}
+}