1

common/html_parser.go

@@ -0,0 +1,34 @@
+package common
+
+import (
+	"os"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type htmlParser struct{}
+
+// Parse html：取 body 文本，标题(h1-h6)前后补空行供分块器识别
+func (p *htmlParser) Parse(path string) (string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(b)))
+	if err != nil {
+		return "", err
+	}
+	var sb strings.Builder
+	doc.Find("body").Children().Each(func(i int, sel *goquery.Selection) {
+		switch goquery.NodeName(sel) {
+		case "h1", "h2", "h3", "h4", "h5", "h6", "p", "div", "li", "pre", "blockquote":
+			text := strings.TrimSpace(sel.Text())
+			if text != "" {
+				sb.WriteString(text)
+				sb.WriteString("\n\n")
+			}
+		}
+	})
+	return sb.String(), nil
+}