fix: Qwen Omni gets its files right - laid out by the app's own inputs (audio before image there, they were swapped), a video as Gradio 5 takes one ({video, subtitles}); the model is told it has the files, so it answers about them

internal/server/ai.go

@@ -474,10 +474,15 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em
 		return err
 	}
 	var conf struct {
+		Components []struct {
+			ID   int    `json:"id"`
+			Type string `json:"type"`
+		} `json:"components"`
 		Dependencies []struct {
 			ID      int     `json:"id"`
 			APIName string  `json:"api_name"`
 			Targets [][]any `json:"targets"`
+			Inputs  []int   `json:"inputs"`
 		} `json:"dependencies"`
 	}
 	err = json.NewDecoder(resp.Body).Decode(&conf)
@@ -486,45 +491,79 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em
 		return errors.New("модель: не прочиталась её конфигурация")
 	}
 	fn, trigger := -1, any(nil)
+	var inputs []int
 	for _, d := range conf.Dependencies {
 		if d.APIName == "chat_predict" {
 			fn = d.ID
+			inputs = d.Inputs
 			if len(d.Targets) > 0 && len(d.Targets[0]) > 0 {
 				trigger = d.Targets[0][0]
 			}
 		}
 	}
+	// Its inputs in order, by kind - the files go where the app has them.
+	// An app that does not say: Qwen Omni's order.
+	kinds := []string{"textbox", "audio", "image", "video", "chatbot", "textbox", "slider", "slider", "slider"}
+	if len(inputs) > 0 {
+		types := map[int]string{}
+		for _, c := range conf.Components {
+			types[c.ID] = c.Type
+		}
+		kinds = kinds[:0:0]
+		for _, id := range inputs {
+			kinds = append(kinds, types[id])
+		}
+	}
 	if fn < 0 {
 		return errors.New("модель: нет функции chat_predict")
 	}
 	last := msgs[len(msgs)-1]
-	media := []any{nil, nil, nil}
+	media := map[string]any{}
 	for _, a := range last.Attachments {
-		if a.Data == "" {
+		kind := strings.SplitN(a.MIME, "/", 2)[0]
+		if a.Data == "" || (kind != "image" && kind != "audio" && kind != "video") {
 			continue
 		}
-		slot := -1
-		switch {
-		case strings.HasPrefix(a.MIME, "image/"):
-			slot = 0
-		case strings.HasPrefix(a.MIME, "audio/"):
-			slot = 1
-		case strings.HasPrefix(a.MIME, "video/"):
-			slot = 2
+		fileData, err := aiGradioUpload(ctx, cfg, hdr, a)
+		if err != nil {
+			return err
 		}
-		if slot >= 0 {
-			fileData, err := aiGradioUpload(ctx, cfg, hdr, a)
-			if err != nil {
-				return err
+		if kind == "video" {
+			// Gradio 5 takes a video with its subtitles.
+			media[kind] = map[string]any{"video": fileData, "subtitles": nil}
+		} else {
+			media[kind] = fileData
+		}
+	}
+	// The values in the inputs' order: the question in the first text box,
+	// the system prompt in the second, the talk in the chatbot, the files,
+	// and temperature, top p and top k in the sliders.
+	data := make([]any, len(kinds))
+	texts, sliders := 0, []any{0.7, 0.8, 20}
+	for i, k := range kinds {
+		switch k {
+		case "textbox":
+			if texts == 0 {
+				data[i] = last.Content
+			} else {
+				data[i] = system
+			}
+			texts++
+		case "image", "audio", "video":
+			data[i] = media[k]
+		case "chatbot":
+			data[i] = msgs[:len(msgs)-1]
+		case "slider":
+			if len(sliders) > 0 {
+				data[i], sliders = sliders[0], sliders[1:]
 			}
-			media[slot] = fileData
 		}
 	}
 	b := make([]byte, 8)
 	_, _ = rand.Read(b)
 	session := hex.EncodeToString(b)
 	jr, err := aiPost(ctx, cfg.URL+"/gradio_api/queue/join", map[string]any{
-		"data":     []any{last.Content, media[0], media[1], media[2], msgs[:len(msgs)-1], system, 0.7, 0.8, 20},
+		"data":     data,
 		"fn_index": fn, "trigger_id": trigger, "event_data": nil, "session_hash": session,
 	}, hdr)
 	if err != nil {
@@ -893,6 +932,18 @@ func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) {
 		in.Context = utf8Safe(in.Context[len(in.Context)-30000:])
 	}
 	system := aiSystem
+	// Pictures, sound and video go to the model itself: it is told so, or
+	// it takes them for something a panel assistant does not do.
+	var media []string
+	for _, f := range in.Attachments {
+		if f.Data != "" {
+			media = append(media, f.Name)
+		}
+	}
+	if len(media) > 0 {
+		system += "\n\nК последнему сообщению пользователя приложены файлы (" + strings.Join(media, ", ") +
+			"): ты получаешь их сам - видишь изображения и видео, слышишь аудио. Отвечай на вопросы о них по существу."
+	}
 	if in.Agent {
 		system += aiAgent
 	}