perf: SSH uploads fetch pieces ahead and downloads post them side by side
This commit is contained in:
1 parent
8d7489b577
commit
6916af4236
3 files changed
+147
-21
No files matched your search
@@ -289,3 +289,49 @@ func (a *Agent) doFSSend(ctx context.Context, env proto.TaskEnvelope) proto.Task
|
||||
data, _ := json.Marshal(map[string]any{"path": args.Path, "size": size})
|
||||
return proto.TaskResult{OK: true, Data: data}
|
||||
}
|
||||
|
||||
// ordered fetches pieces 0..n-1 with up to depth in flight and hands them to
|
||||
// use one by one in their order: a stream that must be written in sequence
|
||||
// (stdin over SSH) still gets its pieces fetched ahead.
|
||||
func ordered(ctx context.Context, n, depth int, fetch func(ctx context.Context, i int) ([]byte, error), use func(i int, b []byte) error) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
type got struct {
|
||||
b []byte
|
||||
err error
|
||||
}
|
||||
slots := make([]chan got, n)
|
||||
for i := range slots {
|
||||
slots[i] = make(chan got, 1)
|
||||
}
|
||||
sem := make(chan struct{}, depth)
|
||||
go func() {
|
||||
for i := 0; i < n; i++ {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
go func(i int) {
|
||||
b, err := fetch(ctx, i)
|
||||
slots[i] <- got{b, err}
|
||||
}(i)
|
||||
}
|
||||
}()
|
||||
for i := 0; i < n; i++ {
|
||||
var g got
|
||||
select {
|
||||
case g = <-slots[i]:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
<-sem
|
||||
if g.err != nil {
|
||||
return g.err
|
||||
}
|
||||
if err := use(i, g.b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user