200 lines
7.1 KiB
Go
200 lines
7.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"html"
|
|
"io"
|
|
"mime"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// fetch: one page by its address, for the assistant with «Поиск» on - to
|
|
// read what a search found. The page comes back as the model wants it: its
|
|
// title and main text, headings and lists kept, scripts, styles and menus
|
|
// gone, cut to fetchMaxChars. Only http(s), and never the server's own
|
|
// network: loopback, private and link-local addresses are refused at the
|
|
// moment of connecting, redirects included.
|
|
|
|
const fetchMaxChars = 12000
|
|
|
|
// fetchAllowPrivate lets tests reach their own server.
|
|
var fetchAllowPrivate = false
|
|
|
|
type fetchedPage struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
Truncated bool `json:"truncated"`
|
|
}
|
|
|
|
func privateIP(ip net.IP) bool {
|
|
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
|
ip.IsUnspecified() || ip.IsMulticast() || ip.IsInterfaceLocalMulticast()
|
|
}
|
|
|
|
var fetchClient = &http.Client{
|
|
Timeout: 20 * time.Second,
|
|
Transport: &http.Transport{
|
|
Proxy: nil,
|
|
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
host, port, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, ip := range ips {
|
|
if privateIP(ip) && !fetchAllowPrivate {
|
|
continue
|
|
}
|
|
var d net.Dialer
|
|
return d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
|
}
|
|
return nil, errors.New("адрес во внутренней сети - не открываю")
|
|
},
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ResponseHeaderTimeout: 15 * time.Second,
|
|
},
|
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
|
if len(via) >= 5 {
|
|
return errors.New("слишком много переадресаций")
|
|
}
|
|
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
|
return errors.New("переадресация не на http(s)")
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
|
|
func fetchPage(ctx context.Context, raw string) (fetchedPage, error) {
|
|
u, err := url.Parse(strings.TrimSpace(raw))
|
|
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
|
return fetchedPage{}, errors.New("нужен адрес http(s)://…")
|
|
}
|
|
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
|
|
req.Header.Set("Accept", "text/html,text/plain,application/json;q=0.9,*/*;q=0.5")
|
|
req.Header.Set("Accept-Language", "ru,en;q=0.8")
|
|
resp, err := fetchClient.Do(req)
|
|
if err != nil {
|
|
return fetchedPage{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode/100 != 2 {
|
|
return fetchedPage{}, errors.New("страница ответила " + resp.Status)
|
|
}
|
|
ct, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
|
|
p := fetchedPage{URL: resp.Request.URL.String(), Type: ct}
|
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 3<<20))
|
|
switch {
|
|
case ct == "text/html" || ct == "application/xhtml+xml" || (ct == "" && strings.Contains(strings.ToLower(string(b[:min(len(b), 512)])), "<html")):
|
|
p.Title, p.Text = pageText(string(b))
|
|
case strings.HasPrefix(ct, "text/") || strings.Contains(ct, "json") || strings.Contains(ct, "xml") || strings.Contains(ct, "javascript"):
|
|
p.Text = strings.TrimSpace(string(b))
|
|
default:
|
|
p.Text = "(двоичный файл " + ct + ", " + strconv.Itoa(len(b)) + " байт - текста нет)"
|
|
}
|
|
if !utf8.ValidString(p.Text) {
|
|
p.Text = strings.ToValidUTF8(p.Text, "")
|
|
}
|
|
if r := []rune(p.Text); len(r) > fetchMaxChars {
|
|
p.Text, p.Truncated = string(r[:fetchMaxChars]), true
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
var (
|
|
reTitle = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
|
reDrop = regexp.MustCompile(`(?is)<(script|style|noscript|svg|nav|footer|header|aside|form|iframe|template|button|select)\b[^>]*>.*?</(script|style|noscript|svg|nav|footer|header|aside|form|iframe|template|button|select)>`)
|
|
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
|
reMain = regexp.MustCompile(`(?is)<(main|article)\b[^>]*>(.*)</(main|article)>`)
|
|
reHead = regexp.MustCompile(`(?is)<h([1-6])\b[^>]*>`)
|
|
reLi = regexp.MustCompile(`(?is)<li\b[^>]*>`)
|
|
reBlock = regexp.MustCompile(`(?is)</?(p|div|section|br|tr|table|ul|ol|pre|blockquote|h[1-6]|dd|dt|figcaption)\b[^>]*>`)
|
|
reCell = regexp.MustCompile(`(?is)</t[dh]>`)
|
|
reSpaces = regexp.MustCompile(`[ \t\x{00a0}]+`)
|
|
reBlanks = regexp.MustCompile(`\n{3,}`)
|
|
rePre = regexp.MustCompile(`(?is)<pre\b[^>]*>.*?</pre>`)
|
|
rePreMark = regexp.MustCompile("\x00pre[0-9]+\x00")
|
|
reWS = regexp.MustCompile(`\s+`)
|
|
)
|
|
|
|
// pageText is a page's title and its text, in Markdown-ish lines.
|
|
func pageText(page string) (string, string) {
|
|
title := ""
|
|
if m := reTitle.FindStringSubmatch(page); m != nil {
|
|
title = strings.TrimSpace(html.UnescapeString(reTags.ReplaceAllString(m[1], "")))
|
|
}
|
|
s := reComment.ReplaceAllString(page, "")
|
|
s = reDrop.ReplaceAllString(s, " ")
|
|
// Line breaks of the source mean nothing outside <pre>: one space. The
|
|
// <pre> blocks keep theirs (held aside meanwhile).
|
|
var pres []string
|
|
s = rePre.ReplaceAllStringFunc(s, func(p string) string {
|
|
pres = append(pres, p)
|
|
return "\x00pre" + strconv.Itoa(len(pres)-1) + "\x00"
|
|
})
|
|
s = reWS.ReplaceAllString(s, " ")
|
|
s = rePreMark.ReplaceAllStringFunc(s, func(m string) string {
|
|
i, _ := strconv.Atoi(strings.Trim(m, "\x00pre"))
|
|
return pres[i]
|
|
})
|
|
// The page's main part, when it marks one.
|
|
if m := reMain.FindStringSubmatch(s); m != nil && len(m[2]) > 200 {
|
|
s = m[2]
|
|
}
|
|
s = reHead.ReplaceAllStringFunc(s, func(t string) string {
|
|
n, _ := strconv.Atoi(reHead.FindStringSubmatch(t)[1])
|
|
return "\n\n" + strings.Repeat("#", n) + " "
|
|
})
|
|
s = reLi.ReplaceAllString(s, "\n- ")
|
|
s = reCell.ReplaceAllString(s, " | ")
|
|
s = reBlock.ReplaceAllString(s, "\n")
|
|
s = html.UnescapeString(reTags.ReplaceAllString(s, ""))
|
|
var lines []string
|
|
for _, l := range strings.Split(s, "\n") {
|
|
l = strings.TrimSpace(reSpaces.ReplaceAllString(l, " "))
|
|
if l == "-" || l == "|" {
|
|
continue
|
|
}
|
|
lines = append(lines, l)
|
|
}
|
|
text := reBlanks.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
|
|
return title, strings.TrimSpace(text)
|
|
}
|
|
|
|
// POST /api/panel/ai/fetch - {url}: the page as text, for a user with the
|
|
// assistant on.
|
|
func (s *Server) handleAIFetch(w http.ResponseWriter, r *http.Request) {
|
|
a := s.currentAdmin(r)
|
|
if !s.aiOn(a.ID) {
|
|
writeJSON(w, map[string]any{"ok": false, "error": "ассистент выключен в профиле"})
|
|
return
|
|
}
|
|
var in struct {
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 8<<10)).Decode(&in); err != nil {
|
|
writeJSON(w, map[string]any{"ok": false, "error": "плохой запрос"})
|
|
return
|
|
}
|
|
p, err := fetchPage(r.Context(), in.URL)
|
|
if err != nil {
|
|
writeJSON(w, map[string]any{"ok": false, "error": err.Error()})
|
|
return
|
|
}
|
|
writeJSON(w, map[string]any{"ok": true, "data": p})
|
|
}
|