package common

import (
	"strings"

	"github.com/go-ego/gse"
)

var seg gse.Segmenter

func init() {
	if err := seg.LoadDict(); err != nil {
		panic("gse load dict failed: " + err.Error())
	}
}

// Tokenize 中文分词，空格连接（FTS5 索引列格式：content_tokens）
func Tokenize(text string) string {
	tokens := seg.Cut(text, true)
	var sb strings.Builder
	for _, t := range tokens {
		t = strings.TrimSpace(t)
		if t == "" {
			continue
		}
		sb.WriteString(t)
		sb.WriteString(" ")
	}
	return strings.TrimSpace(sb.String())
}

// TokenizeQuery FTS5 查询串：每个词用双引号包裹（AND 语义），过滤 FTS5 特殊字符
func TokenizeQuery(text string) string {
	tokens := seg.Cut(text, true)
	var parts []string
	for _, t := range tokens {
		t = strings.TrimSpace(t)
		if t == "" {
			continue
		}
		if strings.ContainsAny(t, "\"*:()") {
			continue
		}
		parts = append(parts, "\""+t+"\"")
	}
	return strings.Join(parts, " ")
}