perf: several dragged items cross to another place as one tar stream; older agents get them one by one
This commit is contained in:
1 parent
6916af4236
commit
7d1996dd7d
8 files changed
+281
-52
No files matched your search
+62
-17
@@ -99,7 +99,24 @@ func treeSize(ctx context.Context, p string) int64 {
|
|||||||
// packTar writes src as a tar stream whose entries start at name. Owners are
|
// packTar writes src as a tar stream whose entries start at name. Owners are
|
||||||
// left out: the numbers mean other people on another machine.
|
// left out: the numbers mean other people on another machine.
|
||||||
func packTar(ctx context.Context, w io.Writer, src, name string, m *shipMeter) error {
|
func packTar(ctx context.Context, w io.Writer, src, name string, m *shipMeter) error {
|
||||||
|
return packTars(ctx, w, []shipItem{{src, name}}, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shipItem is one thing sent: where it is and the name it goes under.
|
||||||
|
type shipItem struct{ src, name string }
|
||||||
|
|
||||||
|
// packTars writes the items as one tar stream, each under its name.
|
||||||
|
func packTars(ctx context.Context, w io.Writer, items []shipItem, m *shipMeter) error {
|
||||||
tw := tar.NewWriter(w)
|
tw := tar.NewWriter(w)
|
||||||
|
for _, it := range items {
|
||||||
|
if err := packInto(ctx, tw, it.src, it.name, m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tw.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func packInto(ctx context.Context, tw *tar.Writer, src, name string, m *shipMeter) error {
|
||||||
err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
|
err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -152,11 +169,8 @@ func packTar(ctx context.Context, w io.Writer, src, name string, m *shipMeter) e
|
|||||||
m.files.Add(1)
|
m.files.Add(1)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return tw.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
type countReader struct {
|
type countReader struct {
|
||||||
r io.Reader
|
r io.Reader
|
||||||
@@ -175,10 +189,24 @@ func (a *Agent) doShipOut(parent context.Context, env proto.TaskEnvelope) proto.
|
|||||||
if err := json.Unmarshal(env.Args, &args); err != nil {
|
if err := json.Unmarshal(env.Args, &args); err != nil {
|
||||||
return proto.TaskResult{Error: err.Error()}
|
return proto.TaskResult{Error: err.Error()}
|
||||||
}
|
}
|
||||||
if err := shipName(args.Name); err != nil {
|
names := args.ShipNames()
|
||||||
|
for _, n := range names {
|
||||||
|
if err := shipName(n); err != nil {
|
||||||
return proto.TaskResult{Error: err.Error()}
|
return proto.TaskResult{Error: err.Error()}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if len(args.More) > 0 && args.SSH != nil {
|
||||||
|
return proto.TaskResult{Error: "по SSH-адресу - по одному"}
|
||||||
|
}
|
||||||
src := filepath.Clean(args.Src)
|
src := filepath.Clean(args.Src)
|
||||||
|
srcs := []string{args.Src}
|
||||||
|
for _, p := range args.More {
|
||||||
|
// The others: of the same folder only.
|
||||||
|
if path.Dir(path.Clean(p)) != path.Dir(path.Clean(args.Src)) {
|
||||||
|
return proto.TaskResult{Error: "несколько - только из одной папки"}
|
||||||
|
}
|
||||||
|
srcs = append(srcs, p)
|
||||||
|
}
|
||||||
var conn *sshfsConn
|
var conn *sshfsConn
|
||||||
if args.Conn != "" {
|
if args.Conn != "" {
|
||||||
// The files are on an SSH host this agent holds a connection to.
|
// The files are on an SSH host this agent holds a connection to.
|
||||||
@@ -195,14 +223,20 @@ func (a *Agent) doShipOut(parent context.Context, env proto.TaskEnvelope) proto.
|
|||||||
m := &shipMeter{}
|
m := &shipMeter{}
|
||||||
m.note.Store("подсчёт")
|
m.note.Store("подсчёт")
|
||||||
go a.reportShip(ctx, args.Job, m, cancel)
|
go a.reportShip(ctx, args.Job, m, cancel)
|
||||||
|
var total int64
|
||||||
|
items := make([]shipItem, len(srcs))
|
||||||
|
for i, p := range srcs {
|
||||||
if conn != nil {
|
if conn != nil {
|
||||||
m.total.Store(a.remoteSize(conn, src))
|
total += a.remoteSize(conn, path.Clean(p))
|
||||||
} else {
|
} else {
|
||||||
m.total.Store(treeSize(ctx, src))
|
total += treeSize(ctx, filepath.Clean(p))
|
||||||
}
|
}
|
||||||
pack := func(w io.Writer) error { return packTar(ctx, w, src, args.Name, m) }
|
items[i] = shipItem{filepath.Clean(p), names[i]}
|
||||||
|
}
|
||||||
|
m.total.Store(total)
|
||||||
|
pack := func(w io.Writer) error { return packTars(ctx, w, items, m) }
|
||||||
if conn != nil {
|
if conn != nil {
|
||||||
pack = func(w io.Writer) error { return a.packRemote(ctx, w, conn, src, args.Name, m) }
|
pack = func(w io.Writer) error { return a.packRemote(ctx, w, conn, srcs, names, m) }
|
||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
@@ -318,9 +352,12 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
|
|||||||
if err := json.Unmarshal(env.Args, &args); err != nil {
|
if err := json.Unmarshal(env.Args, &args); err != nil {
|
||||||
return proto.TaskResult{Error: err.Error()}
|
return proto.TaskResult{Error: err.Error()}
|
||||||
}
|
}
|
||||||
if err := shipName(args.Name); err != nil {
|
names := args.ShipNames()
|
||||||
|
for _, n := range names {
|
||||||
|
if err := shipName(n); err != nil {
|
||||||
return proto.TaskResult{Error: err.Error()}
|
return proto.TaskResult{Error: err.Error()}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
dir := filepath.Clean(args.Dir)
|
dir := filepath.Clean(args.Dir)
|
||||||
var conn *sshfsConn
|
var conn *sshfsConn
|
||||||
if args.Conn != "" {
|
if args.Conn != "" {
|
||||||
@@ -331,8 +368,10 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
|
|||||||
}
|
}
|
||||||
conn, dir = c, path.Clean(args.Dir)
|
conn, dir = c, path.Clean(args.Dir)
|
||||||
} else {
|
} else {
|
||||||
if _, err := os.Lstat(filepath.Join(dir, args.Name)); err == nil && args.Conflict != "replace" && args.Conflict != "skip" {
|
for _, n := range names {
|
||||||
return proto.TaskResult{Error: filepath.Join(dir, args.Name) + " уже существует"}
|
if _, err := os.Lstat(filepath.Join(dir, n)); err == nil && args.Conflict != "replace" && args.Conflict != "skip" {
|
||||||
|
return proto.TaskResult{Error: filepath.Join(dir, n) + " уже существует"}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
return proto.TaskResult{Error: err.Error()}
|
return proto.TaskResult{Error: err.Error()}
|
||||||
@@ -366,7 +405,7 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
if conn != nil {
|
if conn != nil {
|
||||||
skipped, err := a.unpackRemote(conn, pr, dir, args.Name, args.Conflict)
|
skipped, err := a.unpackRemote(conn, pr, dir, names, args.Conflict)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
_, _ = io.Copy(io.Discard, pr)
|
_, _ = io.Copy(io.Discard, pr)
|
||||||
}
|
}
|
||||||
@@ -375,13 +414,17 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
|
|||||||
return proto.TaskResult{Error: err.Error()}
|
return proto.TaskResult{Error: err.Error()}
|
||||||
}
|
}
|
||||||
res := map[string]int{}
|
res := map[string]int{}
|
||||||
if skipped {
|
if skipped > 0 {
|
||||||
res["skipped"] = 1
|
res["skipped"] = skipped
|
||||||
}
|
}
|
||||||
data, _ := json.Marshal(res)
|
data, _ := json.Marshal(res)
|
||||||
return proto.TaskResult{OK: true, Data: data}
|
return proto.TaskResult{OK: true, Data: data}
|
||||||
}
|
}
|
||||||
u := &unpacker{dir: dir, name: args.Name, conflict: args.Conflict}
|
allowed := map[string]bool{}
|
||||||
|
for _, n := range names {
|
||||||
|
allowed[n] = true
|
||||||
|
}
|
||||||
|
u := &unpacker{dir: dir, names: allowed, conflict: args.Conflict}
|
||||||
if os.Geteuid() == 0 {
|
if os.Geteuid() == 0 {
|
||||||
// New things belong to whoever owns the folder they land in.
|
// New things belong to whoever owns the folder they land in.
|
||||||
if info, err := os.Stat(dir); err == nil {
|
if info, err := os.Stat(dir); err == nil {
|
||||||
@@ -412,7 +455,8 @@ func (a *Agent) doShipIn(ctx context.Context, env proto.TaskEnvelope) proto.Task
|
|||||||
// the conflict rule for what is there already: folders merge, and files are
|
// the conflict rule for what is there already: folders merge, and files are
|
||||||
// replaced or kept.
|
// replaced or kept.
|
||||||
type unpacker struct {
|
type unpacker struct {
|
||||||
dir, name, conflict string
|
dir, conflict string
|
||||||
|
names map[string]bool // the top entries the stream may carry
|
||||||
uid, gid int
|
uid, gid int
|
||||||
chown bool
|
chown bool
|
||||||
done, skipped int
|
done, skipped int
|
||||||
@@ -429,7 +473,8 @@ func (u *unpacker) unpack(tr *tar.Reader) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
clean := path.Clean(hdr.Name)
|
clean := path.Clean(hdr.Name)
|
||||||
if clean != u.name && !strings.HasPrefix(clean, u.name+"/") {
|
top, _, _ := strings.Cut(clean, "/")
|
||||||
|
if !u.names[top] || clean == ".." || strings.HasPrefix(clean, "../") {
|
||||||
return fmt.Errorf("в архиве чужой путь %q", hdr.Name)
|
return fmt.Errorf("в архиве чужой путь %q", hdr.Name)
|
||||||
}
|
}
|
||||||
target := filepath.Join(u.dir, filepath.FromSlash(clean))
|
target := filepath.Join(u.dir, filepath.FromSlash(clean))
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Several items of one folder go as one tar and land side by side; an entry
|
||||||
|
// of none of them is refused.
|
||||||
|
func TestShipSeveralInOneTar(t *testing.T) {
|
||||||
|
src, dst := t.TempDir(), t.TempDir()
|
||||||
|
_ = os.WriteFile(filepath.Join(src, "a.txt"), []byte("A"), 0o644)
|
||||||
|
_ = os.MkdirAll(filepath.Join(src, "b"), 0o755)
|
||||||
|
_ = os.WriteFile(filepath.Join(src, "b", "c.txt"), []byte("C"), 0o644)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
items := []shipItem{{filepath.Join(src, "a.txt"), "a.txt"}, {filepath.Join(src, "b"), "b"}}
|
||||||
|
if err := packTars(context.Background(), &buf, items, &shipMeter{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
u := &unpacker{dir: dst, names: map[string]bool{"a.txt": true, "b": true}}
|
||||||
|
if err := u.unpack(tar.NewReader(&buf)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if b, _ := os.ReadFile(filepath.Join(dst, "b", "c.txt")); string(b) != "C" {
|
||||||
|
t.Fatalf("b/c.txt: %q", b)
|
||||||
|
}
|
||||||
|
if b, _ := os.ReadFile(filepath.Join(dst, "a.txt")); string(b) != "A" {
|
||||||
|
t.Fatalf("a.txt: %q", b)
|
||||||
|
}
|
||||||
|
buf.Reset()
|
||||||
|
_ = packTars(context.Background(), &buf, items, &shipMeter{})
|
||||||
|
u = &unpacker{dir: t.TempDir(), names: map[string]bool{"a.txt": true}}
|
||||||
|
if err := u.unpack(tar.NewReader(&buf)); err == nil || !strings.Contains(err.Error(), "чужой путь") {
|
||||||
|
t.Fatalf("b let in: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenameTarSeveral(t *testing.T) {
|
||||||
|
var src bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&src)
|
||||||
|
for _, n := range []string{"a", "b/x", "c", "._a"} {
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: n, Mode: 0o644, Typeflag: tar.TypeReg})
|
||||||
|
}
|
||||||
|
_ = tw.Close()
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := renameTar(&out, &src, map[string]string{"a": "a", "b": "b2"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tr := tar.NewReader(&out)
|
||||||
|
var got []string
|
||||||
|
for {
|
||||||
|
h, err := tr.Next()
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
got = append(got, h.Name)
|
||||||
|
}
|
||||||
|
if strings.Join(got, ",") != "a,b2/x" {
|
||||||
|
t.Fatalf("entries: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+50
-23
@@ -641,11 +641,11 @@ func (a *Agent) doSSHFSPut(ctx context.Context, env proto.TaskEnvelope) proto.Ta
|
|||||||
return proto.TaskResult{OK: true, Data: data}
|
return proto.TaskResult{OK: true, Data: data}
|
||||||
}
|
}
|
||||||
|
|
||||||
// renameTar copies a tar stream, giving its top entry base - and everything
|
// renameTar copies a tar stream, giving each top entry named in names - and
|
||||||
// under it - the name name: what the host's tar packs under the file's own
|
// everything under it - its new name: what the host's tar packs under the
|
||||||
// name arrives under the name chosen for it. Entries other than base (the
|
// file's own name arrives under the name chosen for it. Other entries (the
|
||||||
// "._" companions a macOS tar adds) are left out.
|
// "._" companions a macOS tar adds) are left out.
|
||||||
func renameTar(w io.Writer, r io.Reader, base, name string) error {
|
func renameTar(w io.Writer, r io.Reader, names map[string]string) error {
|
||||||
tr, tw := tar.NewReader(r), tar.NewWriter(w)
|
tr, tw := tar.NewReader(r), tar.NewWriter(w)
|
||||||
for {
|
for {
|
||||||
h, err := tr.Next()
|
h, err := tr.Next()
|
||||||
@@ -661,7 +661,8 @@ func renameTar(w io.Writer, r io.Reader, base, name string) error {
|
|||||||
top, rest = n[:i], n[i:]
|
top, rest = n[:i], n[i:]
|
||||||
}
|
}
|
||||||
// Only base and what is under it - nothing that climbs out by "..".
|
// Only base and what is under it - nothing that climbs out by "..".
|
||||||
if top != base || strings.Contains("/"+rest+"/", "/../") {
|
name, ok := names[top]
|
||||||
|
if !ok || strings.Contains("/"+rest+"/", "/../") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
h.Name = name + rest
|
h.Name = name + rest
|
||||||
@@ -677,7 +678,7 @@ func renameTar(w io.Writer, r io.Reader, base, name string) error {
|
|||||||
|
|
||||||
// packRemote writes src on the connection's host as a tar stream named name,
|
// packRemote writes src on the connection's host as a tar stream named name,
|
||||||
// counting what goes for the progress.
|
// counting what goes for the progress.
|
||||||
func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, src, name string, m *shipMeter) error {
|
func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, srcs, names []string, m *shipMeter) error {
|
||||||
defer c.hold()()
|
defer c.hold()()
|
||||||
s, err := a.sshfsSession(c)
|
s, err := a.sshfsSession(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -688,11 +689,19 @@ func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, src,
|
|||||||
out, _ := s.StdoutPipe()
|
out, _ := s.StdoutPipe()
|
||||||
var errb bytes.Buffer
|
var errb bytes.Buffer
|
||||||
s.Stderr = &limitBuf{b: &errb, max: 4096}
|
s.Stderr = &limitBuf{b: &errb, max: 4096}
|
||||||
src = path.Clean(src)
|
// All from one folder: packed from there by their own names.
|
||||||
if err := s.Start("COPYFILE_DISABLE=1 tar -cf - -C " + shq(path.Dir(src)) + " -- " + shq(path.Base(src))); err != nil {
|
dir := path.Dir(path.Clean(srcs[0]))
|
||||||
|
rename := map[string]string{}
|
||||||
|
cmd := "COPYFILE_DISABLE=1 tar -cf - -C " + shq(dir) + " --"
|
||||||
|
for i, p := range srcs {
|
||||||
|
b := path.Base(path.Clean(p))
|
||||||
|
rename[b] = names[i]
|
||||||
|
cmd += " " + shq(b)
|
||||||
|
}
|
||||||
|
if err := s.Start(cmd); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := renameTar(w, &countReader{r: out, n: &m.done}, path.Base(src), name); err != nil {
|
if err := renameTar(w, &countReader{r: out, n: &m.done}, rename); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := s.Wait(); err != nil {
|
if err := s.Wait(); err != nil {
|
||||||
@@ -721,34 +730,52 @@ func (a *Agent) remoteSize(c *sshfsConn, src string) int64 {
|
|||||||
// unpackRemote puts a tar stream into dir on the connection's host, by the
|
// unpackRemote puts a tar stream into dir on the connection's host, by the
|
||||||
// conflict rule: tar keeps or replaces what is there, an "error" rule
|
// conflict rule: tar keeps or replaces what is there, an "error" rule
|
||||||
// refuses when name is there already.
|
// refuses when name is there already.
|
||||||
func (a *Agent) unpackRemote(c *sshfsConn, r io.Reader, dir, name, conflict string) (skipped bool, err error) {
|
func (a *Agent) unpackRemote(c *sshfsConn, r io.Reader, dir string, names []string, conflict string) (skipped int, err error) {
|
||||||
defer c.hold()()
|
defer c.hold()()
|
||||||
target := strings.TrimSuffix(dir, "/") + "/" + name
|
base := strings.TrimSuffix(dir, "/") + "/"
|
||||||
out, _ := a.sshfsRun(c, "test -e "+shq(target)+" -o -L "+shq(target)+" && echo LPRA_EXISTS; tar --version 2>/dev/null | head -n 1", nil)
|
check := ""
|
||||||
if strings.Contains(string(out), "LPRA_EXISTS") {
|
for _, n := range names {
|
||||||
|
check += "test -e " + shq(base+n) + " -o -L " + shq(base+n) + " && echo " + shq("LPRA_EXISTS "+n) + "; "
|
||||||
|
}
|
||||||
|
out, _ := a.sshfsRun(c, check+"true", nil)
|
||||||
|
keep := map[string]string{}
|
||||||
|
var put []string
|
||||||
|
for _, n := range names {
|
||||||
|
if strings.Contains(string(out), "LPRA_EXISTS "+n+"\n") {
|
||||||
switch conflict {
|
switch conflict {
|
||||||
case "skip":
|
case "skip":
|
||||||
// Left as it was, as on a machine.
|
// Left as it was, as on a machine.
|
||||||
return true, nil
|
skipped++
|
||||||
|
continue
|
||||||
case "replace":
|
case "replace":
|
||||||
default:
|
default:
|
||||||
return false, errors.New(target + " уже существует")
|
return 0, errors.New(base + n + " уже существует")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Only name and what is under it reaches the host's tar.
|
keep[n] = n
|
||||||
|
put = append(put, n)
|
||||||
|
}
|
||||||
|
if len(put) == 0 {
|
||||||
|
return skipped, nil
|
||||||
|
}
|
||||||
|
// Only the items sent (and not kept back) reach the host's tar.
|
||||||
pr, pw := io.Pipe()
|
pr, pw := io.Pipe()
|
||||||
go func() { pw.CloseWithError(renameTar(pw, r, name, name)) }()
|
go func() { pw.CloseWithError(renameTar(pw, r, keep)) }()
|
||||||
defer pr.Close()
|
defer pr.Close()
|
||||||
_, err = a.sshfsRun(c, unpackCommand(dir, name, conflict, strings.Contains(string(out), "GNU tar")), pr)
|
_, err = a.sshfsRun(c, unpackCommand(dir, put, conflict), pr)
|
||||||
return false, err
|
return skipped, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// unpackCommand unpacks a tar stream of name into dir; "replace" removes
|
// unpackCommand unpacks a tar stream of names into dir; "replace" removes
|
||||||
// what is there first, so the thing is replaced whole as on a machine.
|
// what is there first, so each thing is replaced whole as on a machine.
|
||||||
func unpackCommand(dir, name, conflict string, gnu bool) string {
|
func unpackCommand(dir string, names []string, conflict string) string {
|
||||||
cmd := "mkdir -p -- " + shq(dir) + " && cd -- " + shq(dir) + " && tar -xf -"
|
cmd := "mkdir -p -- " + shq(dir) + " && cd -- " + shq(dir) + " && tar -xf -"
|
||||||
if conflict == "replace" {
|
if conflict == "replace" {
|
||||||
cmd = "rm -rf -- " + shq(strings.TrimSuffix(dir, "/")+"/"+name) + " && " + cmd
|
rm := "rm -rf --"
|
||||||
|
for _, n := range names {
|
||||||
|
rm += " " + shq(strings.TrimSuffix(dir, "/")+"/"+n)
|
||||||
|
}
|
||||||
|
cmd = rm + " && " + cmd
|
||||||
}
|
}
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -62,7 +62,7 @@ func TestRenameTarEntries(t *testing.T) {
|
|||||||
}
|
}
|
||||||
_ = tw.Close()
|
_ = tw.Close()
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := renameTar(&out, &src, "old", "new"); err != nil {
|
if err := renameTar(&out, &src, map[string]string{"old": "new"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
tr := tar.NewReader(&out)
|
tr := tar.NewReader(&out)
|
||||||
@@ -90,7 +90,7 @@ func TestRenameTarDropsOtherEntries(t *testing.T) {
|
|||||||
}
|
}
|
||||||
_ = tw.Close()
|
_ = tw.Close()
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := renameTar(&out, &src, "r.txt", "new.txt"); err != nil {
|
if err := renameTar(&out, &src, map[string]string{"r.txt": "new.txt"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
tr := tar.NewReader(&out)
|
tr := tar.NewReader(&out)
|
||||||
@@ -165,7 +165,7 @@ func TestRenameTarKeepsInsideName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
_ = tw.Close()
|
_ = tw.Close()
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := renameTar(&out, &src, "d", "d"); err != nil {
|
if err := renameTar(&out, &src, map[string]string{"d": "d"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
tr := tar.NewReader(&out)
|
tr := tar.NewReader(&out)
|
||||||
@@ -184,11 +184,11 @@ func TestRenameTarKeepsInsideName(t *testing.T) {
|
|||||||
|
|
||||||
// Into an SSH host, "replace" replaces the thing whole, as on a machine.
|
// Into an SSH host, "replace" replaces the thing whole, as on a machine.
|
||||||
func TestUnpackCommandReplaces(t *testing.T) {
|
func TestUnpackCommandReplaces(t *testing.T) {
|
||||||
cmd := unpackCommand("/srv", "d", "replace", true)
|
cmd := unpackCommand("/srv", []string{"d"}, "replace")
|
||||||
if !strings.HasPrefix(cmd, "rm -rf -- '/srv/d' && ") || strings.Contains(cmd, "--overwrite") {
|
if !strings.HasPrefix(cmd, "rm -rf -- '/srv/d' && ") || strings.Contains(cmd, "--overwrite") {
|
||||||
t.Fatalf("replace: %s", cmd)
|
t.Fatalf("replace: %s", cmd)
|
||||||
}
|
}
|
||||||
if cmd := unpackCommand("/srv", "d", "error", false); cmd != "mkdir -p -- '/srv' && cd -- '/srv' && tar -xf -" {
|
if cmd := unpackCommand("/srv", []string{"d"}, "error"); cmd != "mkdir -p -- '/srv' && cd -- '/srv' && tar -xf -" {
|
||||||
t.Fatalf("error: %s", cmd)
|
t.Fatalf("error: %s", cmd)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,6 +5,8 @@
|
|||||||
// outbound requests.
|
// outbound requests.
|
||||||
package proto
|
package proto
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
import "encoding/json"
|
import "encoding/json"
|
||||||
|
|
||||||
// Control poll message types (server -> agent, returned from /api/agent/poll).
|
// Control poll message types (server -> agent, returned from /api/agent/poll).
|
||||||
@@ -372,6 +374,20 @@ type FSShipArgs struct {
|
|||||||
// Conn: the files of this side are on the SSH host of this agent's file
|
// Conn: the files of this side are on the SSH host of this agent's file
|
||||||
// connection (an SSH file pane), not on the machine itself.
|
// connection (an SSH file pane), not on the machine itself.
|
||||||
Conn string `json:"conn,omitempty"`
|
Conn string `json:"conn,omitempty"`
|
||||||
|
// More: other things of Src's folder sent in the same stream under their
|
||||||
|
// own names (several files dragged at once). Agents before 0.2.0 do not
|
||||||
|
// know it; the server sends to them one at a time.
|
||||||
|
More []string `json:"more,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShipNames is what lands at the other end: Name, then More by their names.
|
||||||
|
func (a FSShipArgs) ShipNames() []string {
|
||||||
|
n := []string{a.Name}
|
||||||
|
for _, p := range a.More {
|
||||||
|
p = strings.TrimRight(p, "/")
|
||||||
|
n = append(n, p[strings.LastIndex(p, "/")+1:])
|
||||||
|
}
|
||||||
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShipProgress is what a sending machine reports: bytes of files sent out of
|
// ShipProgress is what a sending machine reports: bytes of files sent out of
|
||||||
|
|||||||
@@ -147,3 +147,14 @@ func TestAIChatNeedsUserOn(t *testing.T) {
|
|||||||
t.Fatalf("on: %d %q", w.Code, b)
|
t.Fatalf("on: %d %q", w.Code, b)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestVersionAtLeast(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
v string
|
||||||
|
want bool
|
||||||
|
}{{"0.2.0", true}, {"0.10.1", true}, {"0.1.9", false}, {"", false}, {"dev", false}, {"1.0", true}} {
|
||||||
|
if got := versionAtLeast(c.v, "0.2.0"); got != c.want {
|
||||||
|
t.Errorf("%q: %v", c.v, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+63
-3
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -119,6 +120,8 @@ type shipReq struct {
|
|||||||
// file pane (its connection id); To is still the agent holding ToConn.
|
// file pane (its connection id); To is still the agent holding ToConn.
|
||||||
FromConn string `json:"from_conn"`
|
FromConn string `json:"from_conn"`
|
||||||
ToConn string `json:"to_conn"`
|
ToConn string `json:"to_conn"`
|
||||||
|
// More: other things of Src's folder sent along, under their own names.
|
||||||
|
More []string `json:"more"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// POST /api/panel/agents/{id}/fs/ship - start sending; the answer names the
|
// POST /api/panel/agents/{id}/fs/ship - start sending; the answer names the
|
||||||
@@ -214,7 +217,27 @@ func oldAgent(res proto.TaskResult) proto.TaskResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq) proto.TaskResult {
|
func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq) proto.TaskResult {
|
||||||
args := proto.FSShipArgs{Src: q.Src, Name: q.Name, Dir: q.Dir, Conflict: q.Conflict, Job: j.id}
|
if len(q.More) > 0 && !s.shipsSeveral(from, q) {
|
||||||
|
// One at a time: an SSH address, or an agent from before 0.2.0.
|
||||||
|
var last proto.TaskResult
|
||||||
|
done, skipped := 0, 0
|
||||||
|
for i, p := range append([]string{q.Src}, q.More...) {
|
||||||
|
one := q
|
||||||
|
one.Src, one.More = p, nil
|
||||||
|
if i > 0 {
|
||||||
|
one.Name = path.Base(path.Clean(p))
|
||||||
|
}
|
||||||
|
if last = s.ship1(ctx, j, from, one); !last.OK {
|
||||||
|
return last
|
||||||
|
}
|
||||||
|
var d struct{ Done, Skipped int }
|
||||||
|
_ = json.Unmarshal(last.Data, &d)
|
||||||
|
done, skipped = done+d.Done, skipped+d.Skipped
|
||||||
|
}
|
||||||
|
data, _ := json.Marshal(map[string]int{"done": done, "skipped": skipped})
|
||||||
|
return proto.TaskResult{OK: true, Data: data}
|
||||||
|
}
|
||||||
|
args := proto.FSShipArgs{Src: q.Src, Name: q.Name, Dir: q.Dir, Conflict: q.Conflict, Job: j.id, More: q.More}
|
||||||
if q.SSH != nil {
|
if q.SSH != nil {
|
||||||
args.SSH, args.Move = q.SSH, q.Move
|
args.SSH, args.Move = q.SSH, q.Move
|
||||||
res, err := s.waitTask(ctx, from, proto.TaskFSShipOut, args, xferTimeout)
|
res, err := s.waitTask(ctx, from, proto.TaskFSShipOut, args, xferTimeout)
|
||||||
@@ -267,9 +290,10 @@ func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq)
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
if q.Move {
|
if q.Move {
|
||||||
task, del := proto.TaskFSDelTree, any(proto.FSPathArgs{Path: q.Src})
|
for _, p := range append([]string{q.Src}, q.More...) {
|
||||||
|
task, del := proto.TaskFSDelTree, any(proto.FSPathArgs{Path: p})
|
||||||
if q.FromConn != "" {
|
if q.FromConn != "" {
|
||||||
raw, _ := json.Marshal(proto.FSPathArgs{Path: q.Src})
|
raw, _ := json.Marshal(proto.FSPathArgs{Path: p})
|
||||||
task, del = proto.TaskSSHFSOp, proto.SSHFSArgs{Conn: q.FromConn, Op: "deltree", Args: raw}
|
task, del = proto.TaskSSHFSOp, proto.SSHFSArgs{Conn: q.FromConn, Op: "deltree", Args: raw}
|
||||||
}
|
}
|
||||||
res, err := s.waitTask(ctx, from, task, del, fsTaskTimeout)
|
res, err := s.waitTask(ctx, from, task, del, fsTaskTimeout)
|
||||||
@@ -280,6 +304,7 @@ func (s *Server) ship1(ctx context.Context, j *shipJob, from string, q shipReq)
|
|||||||
return proto.TaskResult{Error: "отправлено, но здесь не удалилось: " + err.Error()}
|
return proto.TaskResult{Error: "отправлено, но здесь не удалилось: " + err.Error()}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
// The receiver's count: what it put down and what it kept as it was.
|
// The receiver's count: what it put down and what it kept as it was.
|
||||||
return in
|
return in
|
||||||
}
|
}
|
||||||
@@ -434,3 +459,38 @@ func (s *Server) handleRelayGet(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
_, _ = w.Write(data)
|
_, _ = w.Write(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shipsSeveral: both agents take several things in one stream (0.2.0 on).
|
||||||
|
func (s *Server) shipsSeveral(from string, q shipReq) bool {
|
||||||
|
if q.SSH != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, id := range []string{from, q.To} {
|
||||||
|
a, _ := s.db.GetAgent(id)
|
||||||
|
if a == nil || !versionAtLeast(a.Version, "0.2.0") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// versionAtLeast: version v (dotted numbers) is min or newer; anything else
|
||||||
|
// is taken for older.
|
||||||
|
func versionAtLeast(v, min string) bool {
|
||||||
|
a, b := strings.Split(v, "."), strings.Split(min, ".")
|
||||||
|
for i := range b {
|
||||||
|
var x, y int
|
||||||
|
if i < len(a) {
|
||||||
|
n, err := strconv.Atoi(a[i])
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
x = n
|
||||||
|
}
|
||||||
|
y, _ = strconv.Atoi(b[i])
|
||||||
|
if x != y {
|
||||||
|
return x > y
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -155,14 +155,18 @@
|
|||||||
q('.dd-ptext').textContent = 'подготовка…';
|
q('.dd-ptext').textContent = 'подготовка…';
|
||||||
q('.dd-cancel').textContent = 'Стоп';
|
q('.dd-cancel').textContent = 'Стоп';
|
||||||
}
|
}
|
||||||
for (const p of item.paths) {
|
// Between places, all of them go as one stream: the first, and the rest
|
||||||
|
// as "more" (the server sends one by one to older agents).
|
||||||
|
const batch = here ? item.paths.map(p => [p]) : [item.paths];
|
||||||
|
for (const group of batch) {
|
||||||
|
const p = group[0];
|
||||||
if (c.stop) break;
|
if (c.stop) break;
|
||||||
const name = p.split('/').pop();
|
const name = p.split('/').pop();
|
||||||
let r;
|
let r;
|
||||||
if (here) {
|
if (here) {
|
||||||
r = await from.src.op('copy', {src: p, dst: join(t.dir, name), move: isMove, conflict});
|
r = await from.src.op('copy', {src: p, dst: join(t.dir, name), move: isMove, conflict});
|
||||||
} else {
|
} else {
|
||||||
const body = {src: p, name, dir: t.dir, conflict, move: isMove};
|
const body = {src: p, name, dir: t.dir, conflict, move: isMove, more: group.slice(1)};
|
||||||
if (t.source.kind === 'ssh') { body.to = t.pane.agent.id; body.to_conn = t.pane.src.conn; }
|
if (t.source.kind === 'ssh') { body.to = t.pane.agent.id; body.to_conn = t.pane.src.conn; }
|
||||||
else body.to = t.pane.agent.id;
|
else body.to = t.pane.agent.id;
|
||||||
if (from.kind === 'sshfiles') body.from_conn = from.src.conn;
|
if (from.kind === 'sshfiles') body.from_conn = from.src.conn;
|
||||||
@@ -170,7 +174,7 @@
|
|||||||
r = await new MachineSource(from.agent).op('ship', body);
|
r = await new MachineSource(from.agent).op('ship', body);
|
||||||
if (r.ok) {
|
if (r.ok) {
|
||||||
c.job = r.data.job;
|
c.job = r.data.job;
|
||||||
const pre = item.paths.length > 1 ? name + ': ' : '';
|
const pre = '';
|
||||||
r = await FV.followJob(c.job, (pr, text) => {
|
r = await FV.followJob(c.job, (pr, text) => {
|
||||||
if (pr.total > 0 || pr.done > 0) {
|
if (pr.total > 0 || pr.done > 0) {
|
||||||
q('.dd-bar').parentNode.classList.remove('wait');
|
q('.dd-bar').parentNode.classList.remove('wait');
|
||||||
@@ -182,7 +186,7 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!r.ok) { err = r.error || 'ошибка'; break; }
|
if (!r.ok) { err = r.error || 'ошибка'; break; }
|
||||||
done++;
|
done += group.length;
|
||||||
skipped += (r.data && r.data.skipped) || 0;
|
skipped += (r.data && r.data.skipped) || 0;
|
||||||
}
|
}
|
||||||
close();
|
close();
|
||||||
|
|||||||
Reference in new issue
Block a user