fix: Qwen Omni gets its files right - laid out by the app's own inputs (audio before image there, they were swapped), a video as Gradio 5 takes one ({video, subtitles}); the model is told it has the files, so it answers about them

internal/server/ai_test.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -393,3 +394,42 @@ func TestAIChatTitle(t *testing.T) {
 	}
 	t.Fatalf("title: %+v", s.aiChats(a.ID))
 }
+
+// Qwen Omni's files go where its inputs are (audio before image there), and
+// a video as Gradio 5 takes one: {video, subtitles}.
+func TestAIGradioMediaSlots(t *testing.T) {
+	var joined map[string]any
+	n := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/config":
+			fmt.Fprint(w, `{"components":[{"id":20,"type":"textbox"},{"id":16,"type":"audio"},{"id":17,"type":"image"},{"id":18,"type":"video"},`+
+				`{"id":13,"type":"chatbot"},{"id":7,"type":"textbox"},{"id":8,"type":"slider"},{"id":9,"type":"slider"},{"id":10,"type":"slider"}],`+
+				`"dependencies":[{"id":0,"api_name":"chat_predict","targets":[[21,"click"]],"inputs":[20,16,17,18,13,7,8,9,10]}]}`)
+		case "/gradio_api/upload":
+			n++
+			fmt.Fprintf(w, `["/tmp/f%d"]`, n)
+		case "/gradio_api/queue/join":
+			_ = json.NewDecoder(r.Body).Decode(&joined)
+			fmt.Fprint(w, `{"event_id":"e"}`)
+		case "/gradio_api/queue/data":
+			fmt.Fprint(w, "data: {\"msg\":\"process_completed\",\"success\":true,\"output\":{\"data\":[{},{},{},{},[{\"role\":\"assistant\",\"content\":\"ok\"}],{},{}]}}\n\n")
+		}
+	}))
+	defer srv.Close()
+	enc := base64.StdEncoding.EncodeToString([]byte("x"))
+	msgs := []aiMsg{{Role: "user", Content: "что тут", Attachments: []aiAttachment{
+		{Name: "a.png", MIME: "image/png", Data: enc},
+		{Name: "b.mp3", MIME: "audio/mpeg", Data: enc},
+		{Name: "c.mp4", MIME: "video/mp4", Data: enc},
+	}}}
+	if err := aiStream(context.Background(), aiConfig{Type: "gradio", URL: srv.URL}, "sys", msgs, func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+	d := joined["data"].([]any)
+	path := func(v any) string { m, _ := v.(map[string]any); p, _ := m["path"].(string); return p }
+	video, _ := d[3].(map[string]any)
+	if path(d[1]) != "/tmp/f2" || path(d[2]) != "/tmp/f1" || path(video["video"]) != "/tmp/f3" || video["subtitles"] != nil || d[5] != "sys" {
+		t.Fatalf("data: %v", d)
+	}
+}