feat: fetch for the assistant with Поиск on - a page's main text for the model, the server's own network out of reach; shorter words in Настройки and the chat; the + menu's roots a step left. Also files from panes into the chat, docview and drag fixes (by the user)
This commit is contained in:
redtea committed
1 parent
84b42c5369
commit
ae8fffe7d8
17 files changed
+712
-102
No files matched your search
@@ -0,0 +1,199 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"html"
|
||||
"io"
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// fetch: one page by its address, for the assistant with «Поиск» on - to
|
||||
// read what a search found. The page comes back as the model wants it: its
|
||||
// title and main text, headings and lists kept, scripts, styles and menus
|
||||
// gone, cut to fetchMaxChars. Only http(s), and never the server's own
|
||||
// network: loopback, private and link-local addresses are refused at the
|
||||
// moment of connecting, redirects included.
|
||||
|
||||
const fetchMaxChars = 12000
|
||||
|
||||
// fetchAllowPrivate lets tests reach their own server.
|
||||
var fetchAllowPrivate = false
|
||||
|
||||
type fetchedPage struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
|
||||
func privateIP(ip net.IP) bool {
|
||||
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsUnspecified() || ip.IsMulticast() || ip.IsInterfaceLocalMulticast()
|
||||
}
|
||||
|
||||
var fetchClient = &http.Client{
|
||||
Timeout: 20 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if privateIP(ip) && !fetchAllowPrivate {
|
||||
continue
|
||||
}
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||
}
|
||||
return nil, errors.New("адрес во внутренней сети - не открываю")
|
||||
},
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ResponseHeaderTimeout: 15 * time.Second,
|
||||
},
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("слишком много переадресаций")
|
||||
}
|
||||
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
||||
return errors.New("переадресация не на http(s)")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func fetchPage(ctx context.Context, raw string) (fetchedPage, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return fetchedPage{}, errors.New("нужен адрес http(s)://…")
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,text/plain,application/json;q=0.9,*/*;q=0.5")
|
||||
req.Header.Set("Accept-Language", "ru,en;q=0.8")
|
||||
resp, err := fetchClient.Do(req)
|
||||
if err != nil {
|
||||
return fetchedPage{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fetchedPage{}, errors.New("страница ответила " + resp.Status)
|
||||
}
|
||||
ct, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
|
||||
p := fetchedPage{URL: resp.Request.URL.String(), Type: ct}
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 3<<20))
|
||||
switch {
|
||||
case ct == "text/html" || ct == "application/xhtml+xml" || (ct == "" && strings.Contains(strings.ToLower(string(b[:min(len(b), 512)])), "<html")):
|
||||
p.Title, p.Text = pageText(string(b))
|
||||
case strings.HasPrefix(ct, "text/") || strings.Contains(ct, "json") || strings.Contains(ct, "xml") || strings.Contains(ct, "javascript"):
|
||||
p.Text = strings.TrimSpace(string(b))
|
||||
default:
|
||||
p.Text = "(двоичный файл " + ct + ", " + strconv.Itoa(len(b)) + " байт - текста нет)"
|
||||
}
|
||||
if !utf8.ValidString(p.Text) {
|
||||
p.Text = strings.ToValidUTF8(p.Text, "")
|
||||
}
|
||||
if r := []rune(p.Text); len(r) > fetchMaxChars {
|
||||
p.Text, p.Truncated = string(r[:fetchMaxChars]), true
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
var (
|
||||
reTitle = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
||||
reDrop = regexp.MustCompile(`(?is)<(script|style|noscript|svg|nav|footer|header|aside|form|iframe|template|button|select)\b[^>]*>.*?</(script|style|noscript|svg|nav|footer|header|aside|form|iframe|template|button|select)>`)
|
||||
reComment = regexp.MustCompile(`(?s)<!--.*?-->`)
|
||||
reMain = regexp.MustCompile(`(?is)<(main|article)\b[^>]*>(.*)</(main|article)>`)
|
||||
reHead = regexp.MustCompile(`(?is)<h([1-6])\b[^>]*>`)
|
||||
reLi = regexp.MustCompile(`(?is)<li\b[^>]*>`)
|
||||
reBlock = regexp.MustCompile(`(?is)</?(p|div|section|br|tr|table|ul|ol|pre|blockquote|h[1-6]|dd|dt|figcaption)\b[^>]*>`)
|
||||
reCell = regexp.MustCompile(`(?is)</t[dh]>`)
|
||||
reSpaces = regexp.MustCompile(`[ \t\x{00a0}]+`)
|
||||
reBlanks = regexp.MustCompile(`\n{3,}`)
|
||||
rePre = regexp.MustCompile(`(?is)<pre\b[^>]*>.*?</pre>`)
|
||||
rePreMark = regexp.MustCompile("\x00pre[0-9]+\x00")
|
||||
reWS = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// pageText is a page's title and its text, in Markdown-ish lines.
|
||||
func pageText(page string) (string, string) {
|
||||
title := ""
|
||||
if m := reTitle.FindStringSubmatch(page); m != nil {
|
||||
title = strings.TrimSpace(html.UnescapeString(reTags.ReplaceAllString(m[1], "")))
|
||||
}
|
||||
s := reComment.ReplaceAllString(page, "")
|
||||
s = reDrop.ReplaceAllString(s, " ")
|
||||
// Line breaks of the source mean nothing outside <pre>: one space. The
|
||||
// <pre> blocks keep theirs (held aside meanwhile).
|
||||
var pres []string
|
||||
s = rePre.ReplaceAllStringFunc(s, func(p string) string {
|
||||
pres = append(pres, p)
|
||||
return "\x00pre" + strconv.Itoa(len(pres)-1) + "\x00"
|
||||
})
|
||||
s = reWS.ReplaceAllString(s, " ")
|
||||
s = rePreMark.ReplaceAllStringFunc(s, func(m string) string {
|
||||
i, _ := strconv.Atoi(strings.Trim(m, "\x00pre"))
|
||||
return pres[i]
|
||||
})
|
||||
// The page's main part, when it marks one.
|
||||
if m := reMain.FindStringSubmatch(s); m != nil && len(m[2]) > 200 {
|
||||
s = m[2]
|
||||
}
|
||||
s = reHead.ReplaceAllStringFunc(s, func(t string) string {
|
||||
n, _ := strconv.Atoi(reHead.FindStringSubmatch(t)[1])
|
||||
return "\n\n" + strings.Repeat("#", n) + " "
|
||||
})
|
||||
s = reLi.ReplaceAllString(s, "\n- ")
|
||||
s = reCell.ReplaceAllString(s, " | ")
|
||||
s = reBlock.ReplaceAllString(s, "\n")
|
||||
s = html.UnescapeString(reTags.ReplaceAllString(s, ""))
|
||||
var lines []string
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
l = strings.TrimSpace(reSpaces.ReplaceAllString(l, " "))
|
||||
if l == "-" || l == "|" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, l)
|
||||
}
|
||||
text := reBlanks.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
|
||||
return title, strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// POST /api/panel/ai/fetch - {url}: the page as text, for a user with the
|
||||
// assistant on.
|
||||
func (s *Server) handleAIFetch(w http.ResponseWriter, r *http.Request) {
|
||||
a := s.currentAdmin(r)
|
||||
if on, _ := s.db.GetConfig(aiUserKey(a.ID)); on != "1" {
|
||||
writeJSON(w, map[string]any{"ok": false, "error": "ассистент выключен в профиле"})
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 8<<10)).Decode(&in); err != nil {
|
||||
writeJSON(w, map[string]any{"ok": false, "error": "плохой запрос"})
|
||||
return
|
||||
}
|
||||
p, err := fetchPage(r.Context(), in.URL)
|
||||
if err != nil {
|
||||
writeJSON(w, map[string]any{"ok": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]any{"ok": true, "data": p})
|
||||
}
|
||||
Reference in new issue
Block a user