perf: several dragged items cross to another place as one tar stream; older agents get them one by one

internal/agent/sshfs.go

@@ -641,11 +641,11 @@ func (a *Agent) doSSHFSPut(ctx context.Context, env proto.TaskEnvelope) proto.Ta
 	return proto.TaskResult{OK: true, Data: data}
 }
 
-// renameTar copies a tar stream, giving its top entry base - and everything
-// under it - the name name: what the host's tar packs under the file's own
-// name arrives under the name chosen for it. Entries other than base (the
+// renameTar copies a tar stream, giving each top entry named in names - and
+// everything under it - its new name: what the host's tar packs under the
+// file's own name arrives under the name chosen for it. Other entries (the
 // "._" companions a macOS tar adds) are left out.
-func renameTar(w io.Writer, r io.Reader, base, name string) error {
+func renameTar(w io.Writer, r io.Reader, names map[string]string) error {
 	tr, tw := tar.NewReader(r), tar.NewWriter(w)
 	for {
 		h, err := tr.Next()
@@ -661,7 +661,8 @@ func renameTar(w io.Writer, r io.Reader, base, name string) error {
 			top, rest = n[:i], n[i:]
 		}
 		// Only base and what is under it - nothing that climbs out by "..".
-		if top != base || strings.Contains("/"+rest+"/", "/../") {
+		name, ok := names[top]
+		if !ok || strings.Contains("/"+rest+"/", "/../") {
 			continue
 		}
 		h.Name = name + rest
@@ -677,7 +678,7 @@ func renameTar(w io.Writer, r io.Reader, base, name string) error {
 
 // packRemote writes src on the connection's host as a tar stream named name,
 // counting what goes for the progress.
-func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, src, name string, m *shipMeter) error {
+func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, srcs, names []string, m *shipMeter) error {
 	defer c.hold()()
 	s, err := a.sshfsSession(c)
 	if err != nil {
@@ -688,11 +689,19 @@ func (a *Agent) packRemote(ctx context.Context, w io.Writer, c *sshfsConn, src,
 	out, _ := s.StdoutPipe()
 	var errb bytes.Buffer
 	s.Stderr = &limitBuf{b: &errb, max: 4096}
-	src = path.Clean(src)
-	if err := s.Start("COPYFILE_DISABLE=1 tar -cf - -C " + shq(path.Dir(src)) + " -- " + shq(path.Base(src))); err != nil {
+	// All from one folder: packed from there by their own names.
+	dir := path.Dir(path.Clean(srcs[0]))
+	rename := map[string]string{}
+	cmd := "COPYFILE_DISABLE=1 tar -cf - -C " + shq(dir) + " --"
+	for i, p := range srcs {
+		b := path.Base(path.Clean(p))
+		rename[b] = names[i]
+		cmd += " " + shq(b)
+	}
+	if err := s.Start(cmd); err != nil {
 		return err
 	}
-	if err := renameTar(w, &countReader{r: out, n: &m.done}, path.Base(src), name); err != nil {
+	if err := renameTar(w, &countReader{r: out, n: &m.done}, rename); err != nil {
 		return err
 	}
 	if err := s.Wait(); err != nil {
@@ -721,34 +730,52 @@ func (a *Agent) remoteSize(c *sshfsConn, src string) int64 {
 // unpackRemote puts a tar stream into dir on the connection's host, by the
 // conflict rule: tar keeps or replaces what is there, an "error" rule
 // refuses when name is there already.
-func (a *Agent) unpackRemote(c *sshfsConn, r io.Reader, dir, name, conflict string) (skipped bool, err error) {
+func (a *Agent) unpackRemote(c *sshfsConn, r io.Reader, dir string, names []string, conflict string) (skipped int, err error) {
 	defer c.hold()()
-	target := strings.TrimSuffix(dir, "/") + "/" + name
-	out, _ := a.sshfsRun(c, "test -e "+shq(target)+" -o -L "+shq(target)+" && echo LPRA_EXISTS; tar --version 2>/dev/null | head -n 1", nil)
-	if strings.Contains(string(out), "LPRA_EXISTS") {
-		switch conflict {
-		case "skip":
-			// Left as it was, as on a machine.
-			return true, nil
-		case "replace":
-		default:
-			return false, errors.New(target + " уже существует")
-		}
+	base := strings.TrimSuffix(dir, "/") + "/"
+	check := ""
+	for _, n := range names {
+		check += "test -e " + shq(base+n) + " -o -L " + shq(base+n) + " && echo " + shq("LPRA_EXISTS "+n) + "; "
 	}
-	// Only name and what is under it reaches the host's tar.
+	out, _ := a.sshfsRun(c, check+"true", nil)
+	keep := map[string]string{}
+	var put []string
+	for _, n := range names {
+		if strings.Contains(string(out), "LPRA_EXISTS "+n+"\n") {
+			switch conflict {
+			case "skip":
+				// Left as it was, as on a machine.
+				skipped++
+				continue
+			case "replace":
+			default:
+				return 0, errors.New(base + n + " уже существует")
+			}
+		}
+		keep[n] = n
+		put = append(put, n)
+	}
+	if len(put) == 0 {
+		return skipped, nil
+	}
+	// Only the items sent (and not kept back) reach the host's tar.
 	pr, pw := io.Pipe()
-	go func() { pw.CloseWithError(renameTar(pw, r, name, name)) }()
+	go func() { pw.CloseWithError(renameTar(pw, r, keep)) }()
 	defer pr.Close()
-	_, err = a.sshfsRun(c, unpackCommand(dir, name, conflict, strings.Contains(string(out), "GNU tar")), pr)
-	return false, err
+	_, err = a.sshfsRun(c, unpackCommand(dir, put, conflict), pr)
+	return skipped, err
 }
 
-// unpackCommand unpacks a tar stream of name into dir; "replace" removes
-// what is there first, so the thing is replaced whole as on a machine.
-func unpackCommand(dir, name, conflict string, gnu bool) string {
+// unpackCommand unpacks a tar stream of names into dir; "replace" removes
+// what is there first, so each thing is replaced whole as on a machine.
+func unpackCommand(dir string, names []string, conflict string) string {
 	cmd := "mkdir -p -- " + shq(dir) + " && cd -- " + shq(dir) + " && tar -xf -"
 	if conflict == "replace" {
-		cmd = "rm -rf -- " + shq(strings.TrimSuffix(dir, "/")+"/"+name) + " && " + cmd
+		rm := "rm -rf --"
+		for _, n := range names {
+			rm += " " + shq(strings.TrimSuffix(dir, "/")+"/"+n)
+		}
+		cmd = rm + " && " + cmd
 	}
 	return cmd
 }