feat: 拼音写时标注服务(词表最长匹配 + 逐字对齐 JSON)

This commit is contained in:
2026-08-13 13:33:44 +08:00
parent 841191185e
commit 1ce2ab2b24
5 changed files with 184 additions and 0 deletions
+69
View File
@@ -0,0 +1,69 @@
package common
import (
"encoding/json"
"sort"
"strings"
"unicode"
"github.com/mozillazg/go-pinyin"
"36wisdom/biz/consts"
)
// AnnotatePinyin 写时一次性标注:词表最长匹配优先(整体转拼音),未命中逐字转
// pinyin.Tone 带声调);返回逐字对齐的 JSON 数组字符串,非汉字对应空串,
// 下标与原文 rune 一一对应。纯函数,文本未变结果不变(幂等)。
func AnnotatePinyin(text string) string {
runes := []rune(text)
out := make([]string, len(runes))
if len(runes) == 0 {
return "[]"
}
words := make([]string, 0, len(consts.PinyinOverrides))
for w := range consts.PinyinOverrides {
words = append(words, w)
}
sort.Slice(words, func(i, j int) bool { return len(words[i]) > len(words[j]) })
// 词表最长匹配切分:命中片段整体标注(词表拼音音节数须与字数一致)
for i := 0; i < len(runes); {
matched := false
for _, w := range words {
n := len([]rune(w))
if i+n > len(runes) || string(runes[i:i+n]) != w {
continue
}
pys := strings.Fields(consts.PinyinOverrides[w])
if len(pys) != n {
continue
}
copy(out[i:i+n], pys)
i += n
matched = true
break
}
if !matched {
i++
}
}
// 未命中汉字逐字转拼音;pi 与 pinyin.Pinyin 的汉字输出顺序同步推进
args := pinyin.NewArgs()
args.Style = pinyin.Tone
all := pinyin.Pinyin(string(runes), args)
pi := 0
for idx, r := range runes {
if !unicode.Is(unicode.Han, r) {
continue
}
if out[idx] == "" && pi < len(all) && len(all[pi]) > 0 {
out[idx] = all[pi][0]
}
pi++
}
b, _ := json.Marshal(out)
return string(b)
}
+61
View File
@@ -0,0 +1,61 @@
package common
import (
"encoding/json"
"testing"
)
func parse(t *testing.T, s string) []string {
t.Helper()
var out []string
if err := json.Unmarshal([]byte(s), &out); err != nil {
t.Fatalf("拼音 JSON 解析失败: %v", err)
}
return out
}
func TestAnnotatePinyin_IdiomOverride(t *testing.T) {
// 词表命中整体转拼音(带声调)
pys := parse(t, AnnotatePinyin("声东击西"))
want := []string{"shēng", "dōng", "jī", "xī"}
if len(pys) != len(want) {
t.Fatalf("长度 %d != %d: %v", len(pys), len(want), pys)
}
for i := range want {
if pys[i] != want[i] {
t.Fatalf("第 %d 字 %q != %q", i, pys[i], want[i])
}
}
}
func TestAnnotatePinyin_Alignment(t *testing.T) {
// 逐字对齐:标点/非汉字对应空串
pys := parse(t, AnnotatePinyin("小明,你好!"))
want := []string{"xiǎo", "míng", "", "nǐ", "hǎo", ""}
if len(pys) != len(want) {
t.Fatalf("长度 %d != %d: %v", len(pys), len(want), pys)
}
for i := range want {
if pys[i] != want[i] {
t.Fatalf("第 %d 字 %q != %q", i, pys[i], want[i])
}
}
}
func TestAnnotatePinyin_NonHan(t *testing.T) {
pys := parse(t, AnnotatePinyin("123abc"))
if len(pys) != 6 {
t.Fatalf("长度应 6,实际 %d", len(pys))
}
for i, p := range pys {
if p != "" {
t.Fatalf("第 %d 字符应空串,实际 %q", i, p)
}
}
}
func TestAnnotatePinyin_Empty(t *testing.T) {
if AnnotatePinyin("") != "[]" {
t.Fatal("空文本应返回 []")
}
}