48 lines
957 B
Go
48 lines
957 B
Go
package common
|
||
|
||
import (
|
||
"strings"
|
||
|
||
"github.com/go-ego/gse"
|
||
)
|
||
|
||
var seg gse.Segmenter
|
||
|
||
func init() {
|
||
if err := seg.LoadDict(); err != nil {
|
||
panic("gse load dict failed: " + err.Error())
|
||
}
|
||
}
|
||
|
||
// Tokenize 中文分词，空格连接（FTS5 索引列格式：content_tokens）
|
||
func Tokenize(text string) string {
|
||
tokens := seg.Cut(text, true)
|
||
var sb strings.Builder
|
||
for _, t := range tokens {
|
||
t = strings.TrimSpace(t)
|
||
if t == "" {
|
||
continue
|
||
}
|
||
sb.WriteString(t)
|
||
sb.WriteString(" ")
|
||
}
|
||
return strings.TrimSpace(sb.String())
|
||
}
|
||
|
||
// TokenizeQuery FTS5 查询串：每个词用双引号包裹（AND 语义），过滤 FTS5 特殊字符
|
||
func TokenizeQuery(text string) string {
|
||
tokens := seg.Cut(text, true)
|
||
var parts []string
|
||
for _, t := range tokens {
|
||
t = strings.TrimSpace(t)
|
||
if t == "" {
|
||
continue
|
||
}
|
||
if strings.ContainsAny(t, "\"*:()") {
|
||
continue
|
||
}
|
||
parts = append(parts, "\""+t+"\"")
|
||
}
|
||
return strings.Join(parts, " ")
|
||
}
|