From 173f46a145445c0df372e614a1f791b2259b5a0a Mon Sep 17 00:00:00 2001 From: iRedTea Date: Sat, 17 Oct 2026 15:42:37 +0300 Subject: [PATCH] fix: Qwen Omni gets its files right - laid out by the app's own inputs (audio before image there, they were swapped), a video as Gradio 5 takes one ({video, subtitles}); the model is told it has the files, so it answers about them --- internal/server/ai.go | 83 ++++++++++++++++++++++++++++++-------- internal/server/ai_test.go | 40 ++++++++++++++++++ 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/internal/server/ai.go b/internal/server/ai.go index 81a121c..f997be9 100644 --- a/internal/server/ai.go +++ b/internal/server/ai.go @@ -474,10 +474,15 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em return err } var conf struct { + Components []struct { + ID int `json:"id"` + Type string `json:"type"` + } `json:"components"` Dependencies []struct { ID int `json:"id"` APIName string `json:"api_name"` Targets [][]any `json:"targets"` + Inputs []int `json:"inputs"` } `json:"dependencies"` } err = json.NewDecoder(resp.Body).Decode(&conf) @@ -486,45 +491,79 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em return errors.New("модель: не прочиталась её конфигурация") } fn, trigger := -1, any(nil) + var inputs []int for _, d := range conf.Dependencies { if d.APIName == "chat_predict" { fn = d.ID + inputs = d.Inputs if len(d.Targets) > 0 && len(d.Targets[0]) > 0 { trigger = d.Targets[0][0] } } } + // Its inputs in order, by kind - the files go where the app has them. + // An app that does not say: Qwen Omni's order. + kinds := []string{"textbox", "audio", "image", "video", "chatbot", "textbox", "slider", "slider", "slider"} + if len(inputs) > 0 { + types := map[int]string{} + for _, c := range conf.Components { + types[c.ID] = c.Type + } + kinds = kinds[:0:0] + for _, id := range inputs { + kinds = append(kinds, types[id]) + } + } if fn < 0 { return errors.New("модель: нет функции chat_predict") } last := msgs[len(msgs)-1] - media := []any{nil, nil, nil} + media := map[string]any{} for _, a := range last.Attachments { - if a.Data == "" { + kind := strings.SplitN(a.MIME, "/", 2)[0] + if a.Data == "" || (kind != "image" && kind != "audio" && kind != "video") { continue } - slot := -1 - switch { - case strings.HasPrefix(a.MIME, "image/"): - slot = 0 - case strings.HasPrefix(a.MIME, "audio/"): - slot = 1 - case strings.HasPrefix(a.MIME, "video/"): - slot = 2 + fileData, err := aiGradioUpload(ctx, cfg, hdr, a) + if err != nil { + return err } - if slot >= 0 { - fileData, err := aiGradioUpload(ctx, cfg, hdr, a) - if err != nil { - return err + if kind == "video" { + // Gradio 5 takes a video with its subtitles. + media[kind] = map[string]any{"video": fileData, "subtitles": nil} + } else { + media[kind] = fileData + } + } + // The values in the inputs' order: the question in the first text box, + // the system prompt in the second, the talk in the chatbot, the files, + // and temperature, top p and top k in the sliders. + data := make([]any, len(kinds)) + texts, sliders := 0, []any{0.7, 0.8, 20} + for i, k := range kinds { + switch k { + case "textbox": + if texts == 0 { + data[i] = last.Content + } else { + data[i] = system + } + texts++ + case "image", "audio", "video": + data[i] = media[k] + case "chatbot": + data[i] = msgs[:len(msgs)-1] + case "slider": + if len(sliders) > 0 { + data[i], sliders = sliders[0], sliders[1:] } - media[slot] = fileData } } b := make([]byte, 8) _, _ = rand.Read(b) session := hex.EncodeToString(b) jr, err := aiPost(ctx, cfg.URL+"/gradio_api/queue/join", map[string]any{ - "data": []any{last.Content, media[0], media[1], media[2], msgs[:len(msgs)-1], system, 0.7, 0.8, 20}, + "data": data, "fn_index": fn, "trigger_id": trigger, "event_data": nil, "session_hash": session, }, hdr) if err != nil { @@ -893,6 +932,18 @@ func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) { in.Context = utf8Safe(in.Context[len(in.Context)-30000:]) } system := aiSystem + // Pictures, sound and video go to the model itself: it is told so, or + // it takes them for something a panel assistant does not do. + var media []string + for _, f := range in.Attachments { + if f.Data != "" { + media = append(media, f.Name) + } + } + if len(media) > 0 { + system += "\n\nК последнему сообщению пользователя приложены файлы (" + strings.Join(media, ", ") + + "): ты получаешь их сам - видишь изображения и видео, слышишь аудио. Отвечай на вопросы о них по существу." + } if in.Agent { system += aiAgent } diff --git a/internal/server/ai_test.go b/internal/server/ai_test.go index 0d95b36..9b9e3cc 100644 --- a/internal/server/ai_test.go +++ b/internal/server/ai_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -393,3 +394,42 @@ func TestAIChatTitle(t *testing.T) { } t.Fatalf("title: %+v", s.aiChats(a.ID)) } + +// Qwen Omni's files go where its inputs are (audio before image there), and +// a video as Gradio 5 takes one: {video, subtitles}. +func TestAIGradioMediaSlots(t *testing.T) { + var joined map[string]any + n := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/config": + fmt.Fprint(w, `{"components":[{"id":20,"type":"textbox"},{"id":16,"type":"audio"},{"id":17,"type":"image"},{"id":18,"type":"video"},`+ + `{"id":13,"type":"chatbot"},{"id":7,"type":"textbox"},{"id":8,"type":"slider"},{"id":9,"type":"slider"},{"id":10,"type":"slider"}],`+ + `"dependencies":[{"id":0,"api_name":"chat_predict","targets":[[21,"click"]],"inputs":[20,16,17,18,13,7,8,9,10]}]}`) + case "/gradio_api/upload": + n++ + fmt.Fprintf(w, `["/tmp/f%d"]`, n) + case "/gradio_api/queue/join": + _ = json.NewDecoder(r.Body).Decode(&joined) + fmt.Fprint(w, `{"event_id":"e"}`) + case "/gradio_api/queue/data": + fmt.Fprint(w, "data: {\"msg\":\"process_completed\",\"success\":true,\"output\":{\"data\":[{},{},{},{},[{\"role\":\"assistant\",\"content\":\"ok\"}],{},{}]}}\n\n") + } + })) + defer srv.Close() + enc := base64.StdEncoding.EncodeToString([]byte("x")) + msgs := []aiMsg{{Role: "user", Content: "что тут", Attachments: []aiAttachment{ + {Name: "a.png", MIME: "image/png", Data: enc}, + {Name: "b.mp3", MIME: "audio/mpeg", Data: enc}, + {Name: "c.mp4", MIME: "video/mp4", Data: enc}, + }}} + if err := aiStream(context.Background(), aiConfig{Type: "gradio", URL: srv.URL}, "sys", msgs, func(string) {}); err != nil { + t.Fatal(err) + } + d := joined["data"].([]any) + path := func(v any) string { m, _ := v.(map[string]any); p, _ := m["path"].(string); return p } + video, _ := d[3].(map[string]any) + if path(d[1]) != "/tmp/f2" || path(d[2]) != "/tmp/f1" || path(video["video"]) != "/tmp/f3" || video["subtitles"] != nil || d[5] != "sys" { + t.Fatalf("data: %v", d) + } +}