Add multimodal AI chat attachments
This commit is contained in:
redtea committed
1 parent
8aba30a093
commit
84b42c5369
6 files changed
+434
-39
No files matched your search
+198
-16
@@ -5,13 +5,16 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -39,9 +42,17 @@ func aiUserKey(id int64) string { return "ai_user_" + strconv.FormatInt(id, 10)
|
||||
|
||||
type aiConfig struct{ Type, URL, Key, Model string }
|
||||
|
||||
type aiAttachment struct {
|
||||
Name string `json:"name"`
|
||||
MIME string `json:"mime"`
|
||||
Data string `json:"data,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
}
|
||||
|
||||
type aiMsg struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Attachments []aiAttachment `json:"-"`
|
||||
// ThinkMS: how long the model thought before this answer (kept to show it).
|
||||
ThinkMS int64 `json:"think_ms,omitempty"`
|
||||
}
|
||||
@@ -113,11 +124,41 @@ func aiPost(ctx context.Context, url string, body any, hdr map[string]string) (*
|
||||
|
||||
func aiOpenAI(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, emit func(string)) error {
|
||||
all := append([]aiMsg{{Role: "system", Content: system}}, msgs...)
|
||||
apiMessages := make([]map[string]any, 0, len(all))
|
||||
for i, m := range all {
|
||||
var content any = m.Content
|
||||
if i == len(all)-1 && len(m.Attachments) > 0 {
|
||||
blocks := []any{}
|
||||
for _, a := range m.Attachments {
|
||||
if a.Data == "" {
|
||||
continue
|
||||
}
|
||||
mediaURL := "data:" + a.MIME + ";base64," + a.Data
|
||||
switch {
|
||||
case strings.HasPrefix(a.MIME, "image/"):
|
||||
blocks = append(blocks, map[string]any{"type": "image_url", "image_url": map[string]string{"url": mediaURL}})
|
||||
case strings.HasPrefix(a.MIME, "audio/"):
|
||||
format := strings.TrimPrefix(a.MIME, "audio/")
|
||||
if format == "mpeg" || format == "mp3" {
|
||||
format = "mp3"
|
||||
} else if format == "x-wav" || format == "wave" {
|
||||
format = "wav"
|
||||
}
|
||||
blocks = append(blocks, map[string]any{"type": "input_audio", "input_audio": map[string]string{"data": mediaURL, "format": format}})
|
||||
case strings.HasPrefix(a.MIME, "video/"):
|
||||
blocks = append(blocks, map[string]any{"type": "video_url", "video_url": map[string]string{"url": mediaURL}, "fps": 1})
|
||||
}
|
||||
}
|
||||
blocks = append(blocks, map[string]any{"type": "text", "text": m.Content})
|
||||
content = blocks
|
||||
}
|
||||
apiMessages = append(apiMessages, map[string]any{"role": m.Role, "content": content})
|
||||
}
|
||||
hdr := map[string]string{}
|
||||
if cfg.Key != "" {
|
||||
hdr["Authorization"] = "Bearer " + cfg.Key
|
||||
}
|
||||
resp, err := aiPost(ctx, cfg.URL+"/chat/completions", map[string]any{"model": cfg.Model, "stream": true, "messages": all}, hdr)
|
||||
resp, err := aiPost(ctx, cfg.URL+"/chat/completions", map[string]any{"model": cfg.Model, "stream": true, "messages": apiMessages}, hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -156,7 +197,22 @@ func aiAnthropic(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg,
|
||||
if model == "" {
|
||||
model = "claude-sonnet-5-5"
|
||||
}
|
||||
resp, err := aiPost(ctx, cfg.URL+"/v1/messages", map[string]any{"model": model, "max_tokens": 4096, "system": system, "messages": msgs, "stream": true},
|
||||
apiMessages := make([]map[string]any, 0, len(msgs))
|
||||
for i, m := range msgs {
|
||||
var content any = m.Content
|
||||
if i == len(msgs)-1 && len(m.Attachments) > 0 {
|
||||
blocks := []any{map[string]any{"type": "text", "text": m.Content}}
|
||||
for _, a := range m.Attachments {
|
||||
if a.Data == "" || !strings.HasPrefix(a.MIME, "image/") {
|
||||
continue
|
||||
}
|
||||
blocks = append(blocks, map[string]any{"type": "image", "source": map[string]string{"type": "base64", "media_type": a.MIME, "data": a.Data}})
|
||||
}
|
||||
content = blocks
|
||||
}
|
||||
apiMessages = append(apiMessages, map[string]any{"role": m.Role, "content": content})
|
||||
}
|
||||
resp, err := aiPost(ctx, cfg.URL+"/v1/messages", map[string]any{"model": model, "max_tokens": 4096, "system": system, "messages": apiMessages, "stream": true},
|
||||
map[string]string{"x-api-key": cfg.Key, "anthropic-version": "2023-06-01"})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -187,6 +243,47 @@ func aiAnthropic(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg,
|
||||
})
|
||||
}
|
||||
|
||||
func aiGradioUpload(ctx context.Context, cfg aiConfig, hdr map[string]string, a aiAttachment) (map[string]any, error) {
|
||||
data, err := base64.StdEncoding.DecodeString(a.Data)
|
||||
if err != nil {
|
||||
return nil, errors.New("вложение повреждено")
|
||||
}
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
part, err := mw.CreateFormFile("files", path.Base(a.Name))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = part.Write(data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = mw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.URL+"/gradio_api/upload", &body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := aiClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 400))
|
||||
return nil, fmt.Errorf("Gradio не принял файл (%d): %s", resp.StatusCode, strings.TrimSpace(string(msg)))
|
||||
}
|
||||
var paths []string
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&paths); err != nil || len(paths) == 0 || paths[0] == "" {
|
||||
return nil, errors.New("Gradio не вернул путь загруженного файла")
|
||||
}
|
||||
return map[string]any{"path": paths[0], "url": paths[0], "orig_name": path.Base(a.Name), "mime_type": a.MIME, "size": len(data), "meta": map[string]string{"_type": "gradio.FileData"}}, nil
|
||||
}
|
||||
|
||||
// aiGradio talks to a Gradio chat app (chat_predict of the Qwen Omni demo):
|
||||
// the history and the new text go in through the queue, and the chatbot's
|
||||
// history comes back - whole, or as diffs - with the answer growing at its
|
||||
@@ -241,11 +338,33 @@ func aiGradio(ctx context.Context, cfg aiConfig, system string, msgs []aiMsg, em
|
||||
return errors.New("модель: нет функции chat_predict")
|
||||
}
|
||||
last := msgs[len(msgs)-1]
|
||||
media := []any{nil, nil, nil}
|
||||
for _, a := range last.Attachments {
|
||||
if a.Data == "" {
|
||||
continue
|
||||
}
|
||||
slot := -1
|
||||
switch {
|
||||
case strings.HasPrefix(a.MIME, "image/"):
|
||||
slot = 0
|
||||
case strings.HasPrefix(a.MIME, "audio/"):
|
||||
slot = 1
|
||||
case strings.HasPrefix(a.MIME, "video/"):
|
||||
slot = 2
|
||||
}
|
||||
if slot >= 0 {
|
||||
fileData, err := aiGradioUpload(ctx, cfg, hdr, a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
media[slot] = fileData
|
||||
}
|
||||
}
|
||||
b := make([]byte, 8)
|
||||
_, _ = rand.Read(b)
|
||||
session := hex.EncodeToString(b)
|
||||
jr, err := aiPost(ctx, cfg.URL+"/gradio_api/queue/join", map[string]any{
|
||||
"data": []any{last.Content, nil, nil, nil, msgs[:len(msgs)-1], system, 0.7, 0.8, 20},
|
||||
"data": []any{last.Content, media[0], media[1], media[2], msgs[:len(msgs)-1], system, 0.7, 0.8, 20},
|
||||
"fn_index": fn, "trigger_id": trigger, "event_data": nil, "session_hash": session,
|
||||
}, hdr)
|
||||
if err != nil {
|
||||
@@ -520,32 +639,95 @@ func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
var in struct {
|
||||
Message string `json:"message"`
|
||||
Result string `json:"result"`
|
||||
Context string `json:"context"`
|
||||
Agent bool `json:"agent"`
|
||||
Think bool `json:"think"`
|
||||
Search bool `json:"search"`
|
||||
Chat int64 `json:"chat"`
|
||||
Message string `json:"message"`
|
||||
Result string `json:"result"`
|
||||
Context string `json:"context"`
|
||||
Agent bool `json:"agent"`
|
||||
Think bool `json:"think"`
|
||||
Search bool `json:"search"`
|
||||
Chat int64 `json:"chat"`
|
||||
Attachments []aiAttachment `json:"attachments"`
|
||||
}
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in); err != nil || (in.Message == "" && in.Result == "") {
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 24<<20)).Decode(&in); err != nil || (in.Message == "" && in.Result == "" && len(in.Attachments) == 0) {
|
||||
http.Error(w, "плохой запрос", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(in.Attachments) > 6 {
|
||||
http.Error(w, "можно прикрепить не больше 6 файлов", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
cfg := s.aiConfig()
|
||||
total := 0
|
||||
mediaCounts := map[string]int{}
|
||||
for i := range in.Attachments {
|
||||
f := &in.Attachments[i]
|
||||
if len(f.Name) == 0 || len(f.Name) > 512 || len(f.Text) > 120000 {
|
||||
http.Error(w, "неверное вложение", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if f.Data != "" {
|
||||
decoded, err := base64.StdEncoding.DecodeString(f.Data)
|
||||
if err != nil || len(decoded) > 8<<20 {
|
||||
http.Error(w, "вложение повреждено или больше 8 МиБ", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
total += len(decoded)
|
||||
if total > 16<<20 {
|
||||
http.Error(w, "общий размер вложений больше 16 МиБ", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
kind := ""
|
||||
switch {
|
||||
case strings.HasPrefix(f.MIME, "image/"):
|
||||
kind = "image"
|
||||
case strings.HasPrefix(f.MIME, "audio/"):
|
||||
kind = "audio"
|
||||
case strings.HasPrefix(f.MIME, "video/"):
|
||||
kind = "video"
|
||||
default:
|
||||
http.Error(w, "этот формат вложения не поддерживается", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
mediaCounts[kind]++
|
||||
}
|
||||
if f.Text == "" && f.Data == "" {
|
||||
http.Error(w, "пустое вложение", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
if cfg.Type == "anthropic" && (mediaCounts["audio"] > 0 || mediaCounts["video"] > 0) {
|
||||
http.Error(w, "Anthropic API в этой панели принимает только изображения; для аудио и видео используйте OpenAI-compatible API", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if cfg.Type != "openai" && cfg.Type != "anthropic" && (mediaCounts["image"] > 1 || mediaCounts["audio"] > 1 || mediaCounts["video"] > 1) {
|
||||
http.Error(w, "Gradio принимает не больше одного файла каждого медиа-типа за сообщение", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
stored := strings.TrimSpace(in.Message)
|
||||
for _, f := range in.Attachments {
|
||||
stored += "\n\n[Вложение: " + f.Name + "]"
|
||||
if f.Text != "" {
|
||||
stored += "\nСодержимое файла:\n" + f.Text
|
||||
}
|
||||
}
|
||||
stored = strings.TrimSpace(stored)
|
||||
// No chat (or not one of the user's): a new one; its id goes back in
|
||||
// X-Chat-Id.
|
||||
if in.Chat == 0 || !s.aiOwns(a.ID, in.Chat) {
|
||||
in.Chat = s.aiNewChat(a.ID)
|
||||
}
|
||||
w.Header().Set("X-Chat-Id", strconv.FormatInt(in.Chat, 10))
|
||||
fresh := in.Message != "" && len(s.aiHistoryOf(a.ID, in.Chat, 1)) == 0
|
||||
fresh := stored != "" && len(s.aiHistoryOf(a.ID, in.Chat, 1)) == 0
|
||||
s.aiAppendTo(a.ID, in.Chat, "result", in.Result)
|
||||
s.aiAppendTo(a.ID, in.Chat, "user", in.Message)
|
||||
s.aiAppendTo(a.ID, in.Chat, "user", stored)
|
||||
if fresh {
|
||||
// A new chat gets a short name from the model, aside.
|
||||
go s.aiNameChat(a.ID, in.Chat, in.Message)
|
||||
go s.aiNameChat(a.ID, in.Chat, stored)
|
||||
}
|
||||
msgs := aiWindow(s.aiHistoryOf(a.ID, in.Chat, 60), 24000)
|
||||
if len(msgs) > 0 && msgs[len(msgs)-1].Role == "user" {
|
||||
msgs[len(msgs)-1].Attachments = in.Attachments
|
||||
}
|
||||
if len(in.Context) > 30000 {
|
||||
in.Context = utf8Safe(in.Context[len(in.Context)-30000:])
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user