fix: Qwen Omni gets its files right - laid out by the app's own inputs (audio before image there, they were swapped), a video as Gradio 5 takes one ({video, subtitles}); the model is told it has the files, so it answers about them

internal/server/ai.go

@@ -474,10 +474,15 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em
 		return err
 	}
 	var conf struct {
+		Components []struct {
+			ID   int    `json:"id"`
+			Type string `json:"type"`
+		} `json:"components"`
 		Dependencies []struct {
 			ID      int     `json:"id"`
 			APIName string  `json:"api_name"`
 			Targets [][]any `json:"targets"`
+			Inputs  []int   `json:"inputs"`
 		} `json:"dependencies"`
 	}
 	err = json.NewDecoder(resp.Body).Decode(&conf)
@@ -486,45 +491,79 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em
 		return errors.New("модель: не прочиталась её конфигурация")
 	}
 	fn, trigger := -1, any(nil)
+	var inputs []int
 	for _, d := range conf.Dependencies {
 		if d.APIName == "chat_predict" {
 			fn = d.ID
+			inputs = d.Inputs
 			if len(d.Targets) > 0 && len(d.Targets[0]) > 0 {
 				trigger = d.Targets[0][0]
 			}
 		}
 	}
+	// Its inputs in order, by kind - the files go where the app has them.
+	// An app that does not say: Qwen Omni's order.
+	kinds := []string{"textbox", "audio", "image", "video", "chatbot", "textbox", "slider", "slider", "slider"}
+	if len(inputs) > 0 {
+		types := map[int]string{}
+		for _, c := range conf.Components {
+			types[c.ID] = c.Type
+		}
+		kinds = kinds[:0:0]
+		for _, id := range inputs {
+			kinds = append(kinds, types[id])
+		}
+	}
 	if fn < 0 {
 		return errors.New("модель: нет функции chat_predict")
 	}
 	last := msgs[len(msgs)-1]
-	media := []any{nil, nil, nil}
+	media := map[string]any{}
 	for _, a := range last.Attachments {
-		if a.Data == "" {
+		kind := strings.SplitN(a.MIME, "/", 2)[0]
+		if a.Data == "" || (kind != "image" && kind != "audio" && kind != "video") {
 			continue
 		}
-		slot := -1
+		fileData, err := aiGradioUpload(ctx, cfg, hdr, a)
-		switch {
+		if err != nil {
-		case strings.HasPrefix(a.MIME, "image/"):
+			return err
-			slot = 0
-		case strings.HasPrefix(a.MIME, "audio/"):
-			slot = 1
-		case strings.HasPrefix(a.MIME, "video/"):
-			slot = 2
 		}
-		if slot >= 0 {
+		if kind == "video" {
-			fileData, err := aiGradioUpload(ctx, cfg, hdr, a)
+			// Gradio 5 takes a video with its subtitles.
-			if err != nil {
+			media[kind] = map[string]any{"video": fileData, "subtitles": nil}
-				return err
+		} else {
+			media[kind] = fileData
+		}
+	}
+	// The values in the inputs' order: the question in the first text box,
+	// the system prompt in the second, the talk in the chatbot, the files,
+	// and temperature, top p and top k in the sliders.
+	data := make([]any, len(kinds))
+	texts, sliders := 0, []any{0.7, 0.8, 20}
+	for i, k := range kinds {
+		switch k {
+		case "textbox":
+			if texts == 0 {
+				data[i] = last.Content
+			} else {
+				data[i] = system
+			}
+			texts++
+		case "image", "audio", "video":
+			data[i] = media[k]
+		case "chatbot":
+			data[i] = msgs[:len(msgs)-1]
+		case "slider":
+			if len(sliders) > 0 {
+				data[i], sliders = sliders[0], sliders[1:]
 			}
-			media[slot] = fileData
 		}
 	}
 	b := make([]byte, 8)
 	_, _ = rand.Read(b)
 	session := hex.EncodeToString(b)
 	jr, err := aiPost(ctx, cfg.URL+"/gradio_api/queue/join", map[string]any{
-		"data":     []any{last.Content, media[0], media[1], media[2], msgs[:len(msgs)-1], system, 0.7, 0.8, 20},
+		"data":     data,
 		"fn_index": fn, "trigger_id": trigger, "event_data": nil, "session_hash": session,
 	}, hdr)
 	if err != nil {
@@ -893,6 +932,18 @@ func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) {
 		in.Context = utf8Safe(in.Context[len(in.Context)-30000:])
 	}
 	system := aiSystem
+	// Pictures, sound and video go to the model itself: it is told so, or
+	// it takes them for something a panel assistant does not do.
+	var media []string
+	for _, f := range in.Attachments {
+		if f.Data != "" {
+			media = append(media, f.Name)
+		}
+	}
+	if len(media) > 0 {
+		system += "\n\nК последнему сообщению пользователя приложены файлы (" + strings.Join(media, ", ") +
+			"): ты получаешь их сам - видишь изображения и видео, слышишь аудио. Отвечай на вопросы о них по существу."
+	}
 	if in.Agent {
 		system += aiAgent
 	}

internal/server/ai_test.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -393,3 +394,42 @@ func TestAIChatTitle(t *testing.T) {
 	}
 	t.Fatalf("title: %+v", s.aiChats(a.ID))
 }
+
+// Qwen Omni's files go where its inputs are (audio before image there), and
+// a video as Gradio 5 takes one: {video, subtitles}.
+func TestAIGradioMediaSlots(t *testing.T) {
+	var joined map[string]any
+	n := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/config":
+			fmt.Fprint(w, `{"components":[{"id":20,"type":"textbox"},{"id":16,"type":"audio"},{"id":17,"type":"image"},{"id":18,"type":"video"},`+
+				`{"id":13,"type":"chatbot"},{"id":7,"type":"textbox"},{"id":8,"type":"slider"},{"id":9,"type":"slider"},{"id":10,"type":"slider"}],`+
+				`"dependencies":[{"id":0,"api_name":"chat_predict","targets":[[21,"click"]],"inputs":[20,16,17,18,13,7,8,9,10]}]}`)
+		case "/gradio_api/upload":
+			n++
+			fmt.Fprintf(w, `["/tmp/f%d"]`, n)
+		case "/gradio_api/queue/join":
+			_ = json.NewDecoder(r.Body).Decode(&joined)
+			fmt.Fprint(w, `{"event_id":"e"}`)
+		case "/gradio_api/queue/data":
+			fmt.Fprint(w, "data: {\"msg\":\"process_completed\",\"success\":true,\"output\":{\"data\":[{},{},{},{},[{\"role\":\"assistant\",\"content\":\"ok\"}],{},{}]}}\n\n")
+		}
+	}))
+	defer srv.Close()
+	enc := base64.StdEncoding.EncodeToString([]byte("x"))
+	msgs := []aiMsg{{Role: "user", Content: "что тут", Attachments: []aiAttachment{
+		{Name: "a.png", MIME: "image/png", Data: enc},
+		{Name: "b.mp3", MIME: "audio/mpeg", Data: enc},
+		{Name: "c.mp4", MIME: "video/mp4", Data: enc},
+	}}}
+	if err := aiStream(context.Background(), aiConfig{Type: "gradio", URL: srv.URL}, "sys", msgs, func(string) {}); err != nil {
+		t.Fatal(err)
+	}
+	d := joined["data"].([]any)
+	path := func(v any) string { m, _ := v.(map[string]any); p, _ := m["path"].(string); return p }
+	video, _ := d[3].(map[string]any)
+	if path(d[1]) != "/tmp/f2" || path(d[2]) != "/tmp/f1" || path(video["video"]) != "/tmp/f3" || video["subtitles"] != nil || d[5] != "sys" {
+		t.Fatalf("data: %v", d)
+	}
+}