feat: fetch for the assistant with Поиск on - a page's main text for the model, the server's own network out of reach; shorter words in Настройки and the chat; the + menu's roots a step left. Also files from panes into the chat, docview and drag fixes (by the user)

internal/server/fetch_test.go

@@ -0,0 +1,78 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// A page as the model wants it: title, the main text with headings and
+// lists, no scripts, styles or menus.
+func TestPageText(t *testing.T) {
+	page := `<html><head><title>Go 1.24 &amp; more</title><style>p{color:red}</style>
+<meta name="description" content="Release notes"></head><body>
+<nav><a href="/">Home</a> <a href="/doc">Docs</a></nav>
+<main><h1>Go 1.24</h1><p>Generic <b>type aliases</b> are here.</p><script>track()</script>
+<ul><li>Swiss tables</li><li>weak pointers</li></ul></main><footer>© Google</footer></body></html>`
+	title, text := pageText(page)
+	if title != "Go 1.24 & more" {
+		t.Fatalf("title %q", title)
+	}
+	for _, want := range []string{"# Go 1.24", "Generic type aliases are here.", "- Swiss tables", "- weak pointers"} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("no %q in:\n%s", want, text)
+		}
+	}
+	for _, not := range []string{"track()", "color:red", "Home", "© Google"} {
+		if strings.Contains(text, not) {
+			t.Fatalf("%q left in:\n%s", not, text)
+		}
+	}
+}
+
+// The server's own network is out of reach: no loopback, private or local
+// addresses, whatever the name says.
+func TestFetchRefusesPrivate(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "secret") }))
+	defer srv.Close()
+	for _, u := range []string{srv.URL, "http://localhost:1/", "http://10.0.0.1/", "http://169.254.169.254/latest/meta-data/", "file:///etc/passwd", "ftp://x/"} {
+		if _, err := fetchPage(context.Background(), u); err == nil {
+			t.Errorf("%s: fetched", u)
+		}
+	}
+}
+
+func TestFetchPage(t *testing.T) {
+	old := fetchAllowPrivate
+	fetchAllowPrivate = true
+	defer func() { fetchAllowPrivate = old }()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/json" {
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprint(w, `{"a":1}`)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprint(w, "<title>T</title><p>"+strings.Repeat("слово ", 5000)+"</p>")
+	}))
+	defer srv.Close()
+	p, err := fetchPage(context.Background(), srv.URL+"/page")
+	if err != nil || p.Title != "T" || !p.Truncated || len([]rune(p.Text)) > fetchMaxChars+10 {
+		t.Fatalf("page: %v %q %v %d", err, p.Title, p.Truncated, len(p.Text))
+	}
+	p, err = fetchPage(context.Background(), srv.URL+"/json")
+	if err != nil || p.Text != `{"a":1}` {
+		t.Fatalf("json: %+v %v", p, err)
+	}
+}
+
+// Lines broken in the page's source are one paragraph; code keeps its lines.
+func TestPageTextJoinsWrapped(t *testing.T) {
+	_, text := pageText("<p>The latest Go release,\n   version 1.24,\narrives in February.</p><pre>a\nb</pre>")
+	if !strings.Contains(text, "The latest Go release, version 1.24, arrives in February.") || !strings.Contains(text, "a\nb") {
+		t.Fatalf("%q", text)
+	}
+}