Add multimodal AI chat attachments

internal/server/ai_test.go

@@ -99,6 +99,38 @@ func TestAIOpenAI(t *testing.T) {
 	}
 }
 
+func TestAIOpenAIAttachments(t *testing.T) {
+	var body map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewDecoder(r.Body).Decode(&body)
+		fmt.Fprint(w, "data: [DONE]\n\n")
+	}))
+	defer srv.Close()
+	msg := aiMsg{Role: "user", Content: "посмотри", Attachments: []aiAttachment{
+		{Name: "image.png", MIME: "image/png", Data: "AQID"},
+		{Name: "voice.wav", MIME: "audio/wav", Data: "BAUG"},
+		{Name: "clip.mp4", MIME: "video/mp4", Data: "BwgJ"},
+	}}
+	if err := aiOpenAI(context.Background(), aiConfig{Type: "openai", URL: srv.URL, Model: "m"}, "sys", []aiMsg{msg}, func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+	messages := body["messages"].([]any)
+	content := messages[1].(map[string]any)["content"].([]any)
+	if len(content) != 4 || content[3].(map[string]any)["text"] != "посмотри" {
+		t.Fatalf("content: %#v", content)
+	}
+	if got := content[0].(map[string]any)["image_url"].(map[string]any)["url"].(string); !strings.HasPrefix(got, "data:image/png;base64,") {
+		t.Fatalf("image: %q", got)
+	}
+	if got := content[1].(map[string]any)["input_audio"].(map[string]any)["format"]; got != "wav" {
+		t.Fatalf("audio: %#v", content[1])
+	}
+	video := content[2].(map[string]any)
+	if video["type"] != "video_url" || video["fps"] != float64(1) || !strings.HasPrefix(video["video_url"].(map[string]any)["url"].(string), "data:video/mp4;base64,") {
+		t.Fatalf("video: %#v", video)
+	}
+}
+
 func TestAIAnthropic(t *testing.T) {
 	var body map[string]any
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -159,6 +191,43 @@ func TestAIChatNeedsUserOn(t *testing.T) {
 	}
 }
 
+func TestAIChatKeepsAttachmentTextOnly(t *testing.T) {
+	s, c := newTestServer(t)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var in struct {
+			Messages []map[string]any `json:"messages"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&in)
+		content := in.Messages[len(in.Messages)-1]["content"].([]any)
+		if content[0].(map[string]any)["type"] != "image_url" {
+			http.Error(w, "image missing", http.StatusBadRequest)
+			return
+		}
+		fmt.Fprint(w, "data: [DONE]\n\n")
+	}))
+	defer srv.Close()
+	_ = s.db.SetConfig(cfgAIType, "openai")
+	_ = s.db.SetConfig(cfgAIURL, srv.URL)
+	a, _ := s.db.AdminByUsername("adm")
+	_ = s.db.SetConfig(aiUserKey(a.ID), "1")
+	chatID := s.aiNewChat(a.ID)
+	s.aiAppendTo(a.ID, chatID, "user", "предыдущий вопрос")
+	s.aiAppendTo(a.ID, chatID, "assistant", "ответ")
+	body := fmt.Sprintf(`{"message":"посмотри","chat":%d,"attachments":[{"name":"x.png","mime":"image/png","data":"AQID"}]}`, chatID)
+	req := httptest.NewRequest(http.MethodPost, "/api/panel/ai/chat", strings.NewReader(body))
+	req.AddCookie(c)
+	w := httptest.NewRecorder()
+	s.Handler().ServeHTTP(w, req)
+	if w.Code != http.StatusOK {
+		t.Fatalf("response: %d %s", w.Code, w.Body.String())
+	}
+	chat, _ := strconv.ParseInt(w.Header().Get("X-Chat-Id"), 10, 64)
+	history := s.aiHistoryOf(a.ID, chat, 10)
+	if chat != chatID || len(history) != 3 || !strings.Contains(history[2].Content, "x.png") || strings.Contains(history[2].Content, "AQID") {
+		t.Fatalf("history contains wrong attachment data: %+v", history)
+	}
+}
+
 func TestVersionAtLeast(t *testing.T) {
 	for _, c := range []struct {
 		v    string