perf: several dragged items cross to another place as one tar stream; older agents get them one by one

This commit is contained in:
redtea committed 2026-10-15 22:28:46 +03:00
1 parent 6916af4236
commit 7d1996dd7d
8 files changed
+281 -52

No files matched your search

+62 -17
View File
@@ -99,7 +99,24 @@ func treeSize(ctx context.Context, p string) int64 {
// packTar writes src as a tar stream whose entries start at name. Owners are // packTar writes src as a tar stream whose entries start at name. Owners are
// left out: the numbers mean other people on another machine. // left out: the numbers mean other people on another machine.
func packTar(ctx context.Context, w io.Writer, src, name string, m *shipMeter) error { func packTar(ctx context.Context, w io.Writer, src, name string, m *shipMeter) error {
return packTars(ctx, w, []shipItem{{src, name}}, m)
}
// shipItem is one thing sent: where it is and the name it goes under.
type shipItem struct{ src, name string }
// packTars writes the items as one tar stream, each under its name.
func packTars(ctx context.Context, w io.Writer, items []shipItem, m *shipMeter) error {
tw := tar.NewWriter(w) tw := tar.NewWriter(w)
for _, it := range items {
if err := packInto(ctx, tw, it.src, it.name, m); err != nil {
return err
}
}
return tw.Close()
}
func packInto(ctx context.Context, tw *tar.Writer, src, name string, m *shipMeter) error {
err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
@@ -152,11 +169,8 @@ func packTar(ctx context.Context, w io.Writer, src, name string, m *shipMeter) e
m.files.Add(1) m.files.Add(1)
return nil return nil
}) })
if err != nil {
return err return err
} }
return tw.Close()
}
type countReader struct { type countReader struct {
r io.Reader r io.Reader
@@ -175,10 +189,24 @@ func (a *Agent) doShipOut(parent context.Context, env proto.TaskEnvelope) proto.
if err := json.Unmarshal(env.Args, &args); err != nil { if err := json.Unmarshal(env.Args, &args); err != nil {
return proto.TaskResult{Error: err.Error()} return proto.TaskResult{Error: err.Error()}
} }
if err := shipName(args.Name); err != nil { names := args.ShipNames()
for _, n := range names {
if err := shipName(n); err != nil {
return proto.TaskResult{Error: err.Error()} return proto.TaskResult{Error: err.Error()}
} }
}
if len(args.More) > 0 && args.SSH != nil {
return proto.TaskResult{Error: "по SSH-адресу - по одному"}
}
src := filepath.Clean(args.Src) src := filepath.Clean(args.Src)
srcs := []string{args.Src}
for _, p := range args.More {
// The others: of the same folder only.
if path.Dir(path.Clean(p)) != path.Dir(path.Clean(args.Src)) {
return proto.TaskResult{Error: "несколько - только из одной папки"}
}
srcs = append(srcs, p)
}
var conn *sshfsConn var conn *sshfsConn
if args.Conn != "" { if args.Conn != "" {
// The files are on an SSH host this agent holds a connection to. // The files are on an SSH host this agent holds a connection to.
@@ -195,14 +223,20 @@ func (a *Agent) doShipOut(parent context.Context, env proto.TaskEnvelope) proto.
m := &shipMeter{} m := &shipMeter{}
m.note.Store("подсчёт") m.note.Store("подсчёт")
go a.reportShip(ctx, args.Job, m, cancel) go a.reportShip(ctx, args.Job, m, cancel)
var total int64
items := make([]shipItem, len(srcs))
for i, p := range srcs {
if conn != nil { if conn != nil {
m.total.Store(a.remoteSize(conn, src)) total += a.remoteSize(conn, path.Clean(p))
} else { } else {
m.total.Store(treeSize(ctx, src)) total += treeSize(ctx, filepath.Clean(p))
} }
pack := func(w io.Writer) error { return packTar(ctx, w, src, args.Name, m) } items[i] = shipItem{filepath.Clean(p), names[i]}
}
m.total.Store(total)
pack := func(w io.Writer) error { return packTars(ctx, w, items, m) }
if conn != nil { if conn != nil {
pack = func(w io.Writer) error { return a.packRemote(ctx, w, conn, src, args.Name, m) } pack = func(w io.Writer) error { return a.packRemote(ctx, w, conn, srcs, names, m) }
} }
var err error var err error
@@ -318,9 +352,12 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
if err := json.Unmarshal(env.Args, &args); err != nil { if err := json.Unmarshal(env.Args, &args); err != nil {
return proto.TaskResult{Error: err.Error()} return proto.TaskResult{Error: err.Error()}
} }
if err := shipName(args.Name); err != nil { names := args.ShipNames()
for _, n := range names {
if err := shipName(n); err != nil {
return proto.TaskResult{Error: err.Error()} return proto.TaskResult{Error: err.Error()}
} }
}
dir := filepath.Clean(args.Dir) dir := filepath.Clean(args.Dir)
var conn *sshfsConn var conn *sshfsConn
if args.Conn != "" { if args.Conn != "" {
@@ -331,8 +368,10 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
} }
conn, dir = c, path.Clean(args.Dir) conn, dir = c, path.Clean(args.Dir)
} else { } else {
if _, err := os.Lstat(filepath.Join(dir, args.Name)); err == nil && args.Conflict != "replace" && args.Conflict != "skip" { for _, n := range names {
return proto.TaskResult{Error: filepath.Join(dir, args.Name) + " уже существует"} if _, err := os.Lstat(filepath.Join(dir, n)); err == nil && args.Conflict != "replace" && args.Conflict != "skip" {
return proto.TaskResult{Error: filepath.Join(dir, n) + " уже существует"}
}
} }
if err := os.MkdirAll(dir, 0o755); err != nil { if err := os.MkdirAll(dir, 0o755); err != nil {
return proto.TaskResult{Error: err.Error()} return proto.TaskResult{Error: err.Error()}
@@ -366,7 +405,7 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
} }
}() }()
if conn != nil { if conn != nil {
skipped, err := a.unpackRemote(conn, pr, dir, args.Name, args.Conflict) skipped, err := a.unpackRemote(conn, pr, dir, names, args.Conflict)
if err == nil { if err == nil {
_, _ = io.Copy(io.Discard, pr) _, _ = io.Copy(io.Discard, pr)
} }
@@ -375,13 +414,17 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
return proto.TaskResult{Error: err.Error()} return proto.TaskResult{Error: err.Error()}
} }
res := map[string]int{} res := map[string]int{}
if skipped { if skipped > 0 {
res["skipped"] = 1 res["skipped"] = skipped
} }
data, _ := json.Marshal(res) data, _ := json.Marshal(res)
return proto.TaskResult{OK: true, Data: data} return proto.TaskResult{OK: true, Data: data}
} }
u := &unpacker{dir: dir, name: args.Name, conflict: args.Conflict} allowed := map[string]bool{}
for _, n := range names {
allowed[n] = true
}
u := &unpacker{dir: dir, names: allowed, conflict: args.Conflict}
if os.Geteuid() == 0 { if os.Geteuid() == 0 {
// New things belong to whoever owns the folder they land in. // New things belong to whoever owns the folder they land in.
if info, err := os.Stat(dir); err == nil { if info, err := os.Stat(dir); err == nil {
@@ -412,7 +455,8 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
// the conflict rule for what is there already: folders merge, and files are // the conflict rule for what is there already: folders merge, and files are
// replaced or kept. // replaced or kept.
type unpacker struct { type unpacker struct {
dir, name, conflict string dir, conflict string
names map[string]bool // the top entries the stream may carry
uid, gid int uid, gid int
chown bool chown bool
done, skipped int done, skipped int
@@ -429,7 +473,8 @@ func (u *unpacker) unpack(tr *tar.Reader) error {
return err return err
} }
clean := path.Clean(hdr.Name) clean := path.Clean(hdr.Name)
if clean != u.name && !strings.HasPrefix(clean, u.name+"/") { top, _, _ := strings.Cut(clean, "/")
if !u.names[top] || clean == ".." || strings.HasPrefix(clean, "../") {
return fmt.Errorf("в архиве чужой путь %q", hdr.Name) return fmt.Errorf("в архиве чужой путь %q", hdr.Name)
} }
target := filepath.Join(u.dir, filepath.FromSlash(clean)) target := filepath.Join(u.dir, filepath.FromSlash(clean))
+66
View File
@@ -0,0 +1,66 @@
package agent
import (
"archive/tar"
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
// Several items of one folder go as one tar and land side by side; an entry
// of none of them is refused.
func TestShipSeveralInOneTar(t *testing.T) {
src, dst := t.TempDir(), t.TempDir()
_ = os.WriteFile(filepath.Join(src, "a.txt"), []byte("A"), 0o644)
_ = os.MkdirAll(filepath.Join(src, "b"), 0o755)
_ = os.WriteFile(filepath.Join(src, "b", "c.txt"), []byte("C"), 0o644)
var buf bytes.Buffer
items := []shipItem{{filepath.Join(src, "a.txt"), "a.txt"}, {filepath.Join(src, "b"), "b"}}
if err := packTars(context.Background(), &buf, items, &shipMeter{}); err != nil {
t.Fatal(err)
}
u := &unpacker{dir: dst, names: map[string]bool{"a.txt": true, "b": true}}
if err := u.unpack(tar.NewReader(&buf)); err != nil {
t.Fatal(err)
}
if b, _ := os.ReadFile(filepath.Join(dst, "b", "c.txt")); string(b) != "C" {
t.Fatalf("b/c.txt: %q", b)
}
if b, _ := os.ReadFile(filepath.Join(dst, "a.txt")); string(b) != "A" {
t.Fatalf("a.txt: %q", b)
}
buf.Reset()
_ = packTars(context.Background(), &buf, items, &shipMeter{})
u = &unpacker{dir: t.TempDir(), names: map[string]bool{"a.txt": true}}
if err := u.unpack(tar.NewReader(&buf)); err == nil || !strings.Contains(err.Error(), "чужой путь") {
t.Fatalf("b let in: %v", err)
}
}
func TestRenameTarSeveral(t *testing.T) {
var src bytes.Buffer
tw := tar.NewWriter(&src)
for _, n := range []string{"a", "b/x", "c", "._a"} {
_ = tw.WriteHeader(&tar.Header{Name: n, Mode: 0o644, Typeflag: tar.TypeReg})
}
_ = tw.Close()
var out bytes.Buffer
if err := renameTar(&out, &src, map[string]string{"a": "a", "b": "b2"}); err != nil {
t.Fatal(err)
}
tr := tar.NewReader(&out)
var got []string
for {
h, err := tr.Next()
if err != nil {
break
}
got = append(got, h.Name)
}
if strings.Join(got, ",") != "a,b2/x" {
t.Fatalf("entries: %v", got)
}
}
+50 -23
View File
@@ -641,11 +641,11 @@ func (a *Agent) doSSHFSPut(ctx context.Context, env proto.TaskEnvelope) proto.Ta
return proto.TaskResult{OK: true, Data: data} return proto.TaskResult{OK: true, Data: data}
} }
// renameTar copies a tar stream, giving its top entry base - and everything // renameTar copies a tar stream, giving each top entry named in names - and
// under it - the name name: what the host's tar packs under the file's own // everything under it - its new name: what the host's tar packs under the
// name arrives under the name chosen for it. Entries other than base (the // file's own name arrives under the name chosen for it. Other entries (the
// "._" companions a macOS tar adds) are left out. // "._" companions a macOS tar adds) are left out.
func renameTar(w io.Writer, r io.Reader, base, name string) error { func renameTar(w io.Writer, r io.Reader, names map[string]string) error {
tr, tw := tar.NewReader(r), tar.NewWriter(w) tr, tw := tar.NewReader(r), tar.NewWriter(w)
for { for {
h, err := tr.Next() h, err := tr.Next()
@@ -661,7 +661,8 @@ func renameTar(w io.Writer, r io.Reader, base, name string) error {
top, rest = n[:i], n[i:] top, rest = n[:i], n[i:]
} }
// Only base and what is under it - nothing that climbs out by "..". // Only base and what is under it - nothing that climbs out by "..".
if top != base || strings.Contains("/"+rest+"/", "/../") { name, ok := names[top]
if !ok || strings.Contains("/"+rest+"/", "/../") {
continue continue
} }
h.Name = name + rest h.Name = name + rest
@@ -677,7 +678,7 @@ func renameTar(w io.Writer, r io.Reader, base, name string) error {
// packRemote writes src on the connection's host as a tar stream named name, // packRemote writes src on the connection's host as a tar stream named name,
// counting what goes for the progress. // counting what goes for the progress.
func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, src, name string, m *shipMeter) error { func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, srcs, names []string, m *shipMeter) error {
defer c.hold()() defer c.hold()()
s, err := a.sshfsSession(c) s, err := a.sshfsSession(c)
if err != nil { if err != nil {
@@ -688,11 +689,19 @@ func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, src,
out, _ := s.StdoutPipe() out, _ := s.StdoutPipe()
var errb bytes.Buffer var errb bytes.Buffer
s.Stderr = &limitBuf{b: &errb, max: 4096} s.Stderr = &limitBuf{b: &errb, max: 4096}
src = path.Clean(src) // All from one folder: packed from there by their own names.
if err := s.Start("COPYFILE_DISABLE=1 tar -cf - -C " + shq(path.Dir(src)) + " -- " + shq(path.Base(src))); err != nil { dir := path.Dir(path.Clean(srcs[0]))
rename := map[string]string{}
cmd := "COPYFILE_DISABLE=1 tar -cf - -C " + shq(dir) + " --"
for i, p := range srcs {
b := path.Base(path.Clean(p))
rename[b] = names[i]
cmd += " " + shq(b)
}
if err := s.Start(cmd); err != nil {
return err return err
} }
if err := renameTar(w, &countReader{r: out, n: &m.done}, path.Base(src), name); err != nil { if err := renameTar(w, &countReader{r: out, n: &m.done}, rename); err != nil {
return err return err
} }
if err := s.Wait(); err != nil { if err := s.Wait(); err != nil {
@@ -721,34 +730,52 @@ func (a *Agent) remoteSize(c *sshfsConn, src string) int64 {
// unpackRemote puts a tar stream into dir on the connection's host, by the // unpackRemote puts a tar stream into dir on the connection's host, by the
// conflict rule: tar keeps or replaces what is there, an "error" rule // conflict rule: tar keeps or replaces what is there, an "error" rule
// refuses when name is there already. // refuses when name is there already.
func (a *Agent) unpackRemote(c *sshfsConn, r io.Reader, dir, name, conflict string) (skipped bool, err error) { func (a *Agent) unpackRemote(c *sshfsConn, r io.Reader, dir string, names []string, conflict string) (skipped int, err error) {
defer c.hold()() defer c.hold()()
target := strings.TrimSuffix(dir, "/") + "/" + name base := strings.TrimSuffix(dir, "/") + "/"
out, _ := a.sshfsRun(c, "test -e "+shq(target)+" -o -L "+shq(target)+" && echo LPRA_EXISTS; tar --version 2>/dev/null | head -n 1", nil) check := ""
if strings.Contains(string(out), "LPRA_EXISTS") { for _, n := range names {
check += "test -e " + shq(base+n) + " -o -L " + shq(base+n) + " && echo " + shq("LPRA_EXISTS "+n) + "; "
}
out, _ := a.sshfsRun(c, check+"true", nil)
keep := map[string]string{}
var put []string
for _, n := range names {
if strings.Contains(string(out), "LPRA_EXISTS "+n+"\n") {
switch conflict { switch conflict {
case "skip": case "skip":
// Left as it was, as on a machine. // Left as it was, as on a machine.
return true, nil skipped++
continue
case "replace": case "replace":
default: default:
return false, errors.New(target + " уже существует") return 0, errors.New(base + n + " уже существует")
} }
} }
// Only name and what is under it reaches the host's tar. keep[n] = n
put = append(put, n)
}
if len(put) == 0 {
return skipped, nil
}
// Only the items sent (and not kept back) reach the host's tar.
pr, pw := io.Pipe() pr, pw := io.Pipe()
go func() { pw.CloseWithError(renameTar(pw, r, name, name)) }() go func() { pw.CloseWithError(renameTar(pw, r, keep)) }()
defer pr.Close() defer pr.Close()
_, err = a.sshfsRun(c, unpackCommand(dir, name, conflict, strings.Contains(string(out), "GNU tar")), pr) _, err = a.sshfsRun(c, unpackCommand(dir, put, conflict), pr)
return false, err return skipped, err
} }
// unpackCommand unpacks a tar stream of name into dir; "replace" removes // unpackCommand unpacks a tar stream of names into dir; "replace" removes
// what is there first, so the thing is replaced whole as on a machine. // what is there first, so each thing is replaced whole as on a machine.
func unpackCommand(dir, name, conflict string, gnu bool) string { func unpackCommand(dir string, names []string, conflict string) string {
cmd := "mkdir -p -- " + shq(dir) + " && cd -- " + shq(dir) + " && tar -xf -" cmd := "mkdir -p -- " + shq(dir) + " && cd -- " + shq(dir) + " && tar -xf -"
if conflict == "replace" { if conflict == "replace" {
cmd = "rm -rf -- " + shq(strings.TrimSuffix(dir, "/")+"/"+name) + " && " + cmd rm := "rm -rf --"
for _, n := range names {
rm += " " + shq(strings.TrimSuffix(dir, "/")+"/"+n)
}
cmd = rm + " && " + cmd
} }
return cmd return cmd
} }
+5 -5
View File
@@ -62,7 +62,7 @@ func TestRenameTarEntries(t *testing.T) {
} }
_ = tw.Close() _ = tw.Close()
var out bytes.Buffer var out bytes.Buffer
if err := renameTar(&out, &src, "old", "new"); err != nil { if err := renameTar(&out, &src, map[string]string{"old": "new"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
tr := tar.NewReader(&out) tr := tar.NewReader(&out)
@@ -90,7 +90,7 @@ func TestRenameTarDropsOtherEntries(t *testing.T) {
} }
_ = tw.Close() _ = tw.Close()
var out bytes.Buffer var out bytes.Buffer
if err := renameTar(&out, &src, "r.txt", "new.txt"); err != nil { if err := renameTar(&out, &src, map[string]string{"r.txt": "new.txt"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
tr := tar.NewReader(&out) tr := tar.NewReader(&out)
@@ -165,7 +165,7 @@ func TestRenameTarKeepsInsideName(t *testing.T) {
} }
_ = tw.Close() _ = tw.Close()
var out bytes.Buffer var out bytes.Buffer
if err := renameTar(&out, &src, "d", "d"); err != nil { if err := renameTar(&out, &src, map[string]string{"d": "d"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
tr := tar.NewReader(&out) tr := tar.NewReader(&out)
@@ -184,11 +184,11 @@ func TestRenameTarKeepsInsideName(t *testing.T) {
// Into an SSH host, "replace" replaces the thing whole, as on a machine. // Into an SSH host, "replace" replaces the thing whole, as on a machine.
func TestUnpackCommandReplaces(t *testing.T) { func TestUnpackCommandReplaces(t *testing.T) {
cmd := unpackCommand("/srv", "d", "replace", true) cmd := unpackCommand("/srv", []string{"d"}, "replace")
if !strings.HasPrefix(cmd, "rm -rf -- '/srv/d' && ") || strings.Contains(cmd, "--overwrite") { if !strings.HasPrefix(cmd, "rm -rf -- '/srv/d' && ") || strings.Contains(cmd, "--overwrite") {
t.Fatalf("replace: %s", cmd) t.Fatalf("replace: %s", cmd)
} }
if cmd := unpackCommand("/srv", "d", "error", false); cmd != "mkdir -p -- '/srv' && cd -- '/srv' && tar -xf -" { if cmd := unpackCommand("/srv", []string{"d"}, "error"); cmd != "mkdir -p -- '/srv' && cd -- '/srv' && tar -xf -" {
t.Fatalf("error: %s", cmd) t.Fatalf("error: %s", cmd)
} }
} }
+16
View File
@@ -5,6 +5,8 @@
// outbound requests. // outbound requests.
package proto package proto
import "strings"
import "encoding/json" import "encoding/json"
// Control poll message types (server -> agent, returned from /api/agent/poll). // Control poll message types (server -> agent, returned from /api/agent/poll).
@@ -372,6 +374,20 @@ type FSShipArgs struct {
// Conn: the files of this side are on the SSH host of this agent's file // Conn: the files of this side are on the SSH host of this agent's file
// connection (an SSH file pane), not on the machine itself. // connection (an SSH file pane), not on the machine itself.
Conn string `json:"conn,omitempty"` Conn string `json:"conn,omitempty"`
// More: other things of Src's folder sent in the same stream under their
// own names (several files dragged at once). Agents before 0.2.0 do not
// know it; the server sends to them one at a time.
More []string `json:"more,omitempty"`
}
// ShipNames is what lands at the other end: Name, then More by their names.
func (a FSShipArgs) ShipNames() []string {
n := []string{a.Name}
for _, p := range a.More {
p = strings.TrimRight(p, "/")
n = append(n, p[strings.LastIndex(p, "/")+1:])
}
return n
} }
// ShipProgress is what a sending machine reports: bytes of files sent out of // ShipProgress is what a sending machine reports: bytes of files sent out of
+11
View File
@@ -147,3 +147,14 @@ func TestAIChatNeedsUserOn(t *testing.T) {
t.Fatalf("on: %d %q", w.Code, b) t.Fatalf("on: %d %q", w.Code, b)
} }
} }
func TestVersionAtLeast(t *testing.T) {
for _, c := range []struct {
v string
want bool
}{{"0.2.0", true}, {"0.10.1", true}, {"0.1.9", false}, {"", false}, {"dev", false}, {"1.0", true}} {
if got := versionAtLeast(c.v, "0.2.0"); got != c.want {
t.Errorf("%q: %v", c.v, got)
}
}
}
+63 -3
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"net/http" "net/http"
"path"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -119,6 +120,8 @@ type shipReq struct {
// file pane (its connection id); To is still the agent holding ToConn. // file pane (its connection id); To is still the agent holding ToConn.
FromConn string `json:"from_conn"` FromConn string `json:"from_conn"`
ToConn string `json:"to_conn"` ToConn string `json:"to_conn"`
// More: other things of Src's folder sent along, under their own names.
More []string `json:"more"`
} }
// POST /api/panel/agents/{id}/fs/ship - start sending; the answer names the // POST /api/panel/agents/{id}/fs/ship - start sending; the answer names the
@@ -214,7 +217,27 @@ func oldAgent(res proto.TaskResult) proto.TaskResult {
} }
func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq) proto.TaskResult { func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq) proto.TaskResult {
args := proto.FSShipArgs{Src: q.Src, Name: q.Name, Dir: q.Dir, Conflict: q.Conflict, Job: j.id} if len(q.More) > 0 && !s.shipsSeveral(from, q) {
// One at a time: an SSH address, or an agent from before 0.2.0.
var last proto.TaskResult
done, skipped := 0, 0
for i, p := range append([]string{q.Src}, q.More...) {
one := q
one.Src, one.More = p, nil
if i > 0 {
one.Name = path.Base(path.Clean(p))
}
if last = s.ship1(ctx, j, from, one); !last.OK {
return last
}
var d struct{ Done, Skipped int }
_ = json.Unmarshal(last.Data, &d)
done, skipped = done+d.Done, skipped+d.Skipped
}
data, _ := json.Marshal(map[string]int{"done": done, "skipped": skipped})
return proto.TaskResult{OK: true, Data: data}
}
args := proto.FSShipArgs{Src: q.Src, Name: q.Name, Dir: q.Dir, Conflict: q.Conflict, Job: j.id, More: q.More}
if q.SSH != nil { if q.SSH != nil {
args.SSH, args.Move = q.SSH, q.Move args.SSH, args.Move = q.SSH, q.Move
res, err := s.waitTask(ctx, from, proto.TaskFSShipOut, args, xferTimeout) res, err := s.waitTask(ctx, from, proto.TaskFSShipOut, args, xferTimeout)
@@ -267,9 +290,10 @@ func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq)
return out return out
} }
if q.Move { if q.Move {
task, del := proto.TaskFSDelTree, any(proto.FSPathArgs{Path: q.Src}) for _, p := range append([]string{q.Src}, q.More...) {
task, del := proto.TaskFSDelTree, any(proto.FSPathArgs{Path: p})
if q.FromConn != "" { if q.FromConn != "" {
raw, _ := json.Marshal(proto.FSPathArgs{Path: q.Src}) raw, _ := json.Marshal(proto.FSPathArgs{Path: p})
task, del = proto.TaskSSHFSOp, proto.SSHFSArgs{Conn: q.FromConn, Op: "deltree", Args: raw} task, del = proto.TaskSSHFSOp, proto.SSHFSArgs{Conn: q.FromConn, Op: "deltree", Args: raw}
} }
res, err := s.waitTask(ctx, from, task, del, fsTaskTimeout) res, err := s.waitTask(ctx, from, task, del, fsTaskTimeout)
@@ -280,6 +304,7 @@ func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq)
return proto.TaskResult{Error: "отправлено, но здесь не удалилось: " + err.Error()} return proto.TaskResult{Error: "отправлено, но здесь не удалилось: " + err.Error()}
} }
} }
}
// The receiver's count: what it put down and what it kept as it was. // The receiver's count: what it put down and what it kept as it was.
return in return in
} }
@@ -434,3 +459,38 @@ func (s *Server) handleRelayGet(w http.ResponseWriter, r *http.Request) {
} }
_, _ = w.Write(data) _, _ = w.Write(data)
} }
// shipsSeveral: both agents take several things in one stream (0.2.0 on).
func (s *Server) shipsSeveral(from string, q shipReq) bool {
if q.SSH != nil {
return false
}
for _, id := range []string{from, q.To} {
a, _ := s.db.GetAgent(id)
if a == nil || !versionAtLeast(a.Version, "0.2.0") {
return false
}
}
return true
}
// versionAtLeast: version v (dotted numbers) is min or newer; anything else
// is taken for older.
func versionAtLeast(v, min string) bool {
a, b := strings.Split(v, "."), strings.Split(min, ".")
for i := range b {
var x, y int
if i < len(a) {
n, err := strconv.Atoi(a[i])
if err != nil {
return false
}
x = n
}
y, _ = strconv.Atoi(b[i])
if x != y {
return x > y
}
}
return true
}
+8 -4
View File
@@ -155,14 +155,18 @@
q('.dd-ptext').textContent = 'подготовка…'; q('.dd-ptext').textContent = 'подготовка…';
q('.dd-cancel').textContent = 'Стоп'; q('.dd-cancel').textContent = 'Стоп';
} }
for (const p of item.paths) { // Between places, all of them go as one stream: the first, and the rest
// as "more" (the server sends one by one to older agents).
const batch = here ? item.paths.map(p => [p]) : [item.paths];
for (const group of batch) {
const p = group[0];
if (c.stop) break; if (c.stop) break;
const name = p.split('/').pop(); const name = p.split('/').pop();
let r; let r;
if (here) { if (here) {
r = await from.src.op('copy', {src: p, dst: join(t.dir, name), move: isMove, conflict}); r = await from.src.op('copy', {src: p, dst: join(t.dir, name), move: isMove, conflict});
} else { } else {
const body = {src: p, name, dir: t.dir, conflict, move: isMove}; const body = {src: p, name, dir: t.dir, conflict, move: isMove, more: group.slice(1)};
if (t.source.kind === 'ssh') { body.to = t.pane.agent.id; body.to_conn = t.pane.src.conn; } if (t.source.kind === 'ssh') { body.to = t.pane.agent.id; body.to_conn = t.pane.src.conn; }
else body.to = t.pane.agent.id; else body.to = t.pane.agent.id;
if (from.kind === 'sshfiles') body.from_conn = from.src.conn; if (from.kind === 'sshfiles') body.from_conn = from.src.conn;
@@ -170,7 +174,7 @@
r = await new MachineSource(from.agent).op('ship', body); r = await new MachineSource(from.agent).op('ship', body);
if (r.ok) { if (r.ok) {
c.job = r.data.job; c.job = r.data.job;
const pre = item.paths.length > 1 ? name + ': ' : ''; const pre = '';
r = await FV.followJob(c.job, (pr, text) => { r = await FV.followJob(c.job, (pr, text) => {
if (pr.total > 0 || pr.done > 0) { if (pr.total > 0 || pr.done > 0) {
q('.dd-bar').parentNode.classList.remove('wait'); q('.dd-bar').parentNode.classList.remove('wait');
@@ -182,7 +186,7 @@
} }
} }
if (!r.ok) { err = r.error || 'ошибка'; break; } if (!r.ok) { err = r.error || 'ошибка'; break; }
done++; done += group.length;
skipped += (r.data && r.data.skipped) || 0; skipped += (r.data && r.data.skipped) || 0;
} }
close(); close();