Skip to content

Commit 42dd948

Browse files
committed
feat(cloudimg): parallel Range download for image pull
Single-stream GET was the bottleneck for multi-GB cloud images. Probe Range support, then split the blob across PullConns concurrent Range connections into a preallocated file, re-hashing on disk afterwards; fall back to the existing serial stream when the server lacks Range.
1 parent 80ad17e commit 42dd948

9 files changed

Lines changed: 518 additions & 27 deletions

File tree

cmd/core/init.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ func InitImageBackendsForPull(ctx context.Context, conf *config.Config) (*oci.OC
5757
if err != nil {
5858
return nil, nil, fmt.Errorf("init oci backend: %w", err)
5959
}
60-
cloudimgStore, err := cloudimg.New(ctx, conf.RootDir)
60+
cloudimgStore, err := cloudimg.New(ctx, conf.RootDir, conf.EffectivePullConns())
6161
if err != nil {
6262
return nil, nil, fmt.Errorf("init cloudimg backend: %w", err)
6363
}

cmd/images/import.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func (h Handler) importFromReader(ctx context.Context, conf *config.Config, name
102102

103103
func (h Handler) importCloudimgFiles(ctx context.Context, conf *config.Config, name string, files ...string) error {
104104
logger := log.WithFunc("cmd.images.importCloudimgFiles")
105-
cloudimgStore, err := cloudimg.New(ctx, conf.RootDir)
105+
cloudimgStore, err := cloudimg.New(ctx, conf.RootDir, conf.EffectivePullConns())
106106
if err != nil {
107107
return fmt.Errorf("init cloudimg backend: %w", err)
108108
}
@@ -154,7 +154,7 @@ func (h Handler) importOCIFiles(ctx context.Context, conf *config.Config, name s
154154

155155
func (h Handler) importCloudimgReader(ctx context.Context, conf *config.Config, name string, r io.Reader) error {
156156
logger := log.WithFunc("cmd.images.importCloudimgReader")
157-
cloudimgStore, err := cloudimg.New(ctx, conf.RootDir)
157+
cloudimgStore, err := cloudimg.New(ctx, conf.RootDir, conf.EffectivePullConns())
158158
if err != nil {
159159
return fmt.Errorf("init cloudimg backend: %w", err)
160160
}

cmd/root.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ var (
6565
viper.SetDefault("dns", "8.8.8.8,1.1.1.1")
6666
viper.SetDefault("stop_timeout_seconds", 30)
6767
viper.SetDefault("pool_size", runtime.NumCPU())
68+
viper.SetDefault("pull_conns", 8)
6869
viper.SetDefault("log.level", "info")
6970
viper.SetDefault("log.max_size", 500)
7071
viper.SetDefault("log.max_age", 28)

config/config.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ import (
1313
const (
1414
HypervisorCH HypervisorType = "cloud-hypervisor"
1515
HypervisorFirecracker HypervisorType = "firecracker"
16+
17+
// defaultPullConns is the default concurrent Range connections per cloud-image download.
18+
defaultPullConns = 8
1619
)
1720

1821
// HypervisorType identifies the selected hypervisor backend.
@@ -36,6 +39,8 @@ type Config struct {
3639
StopTimeoutSeconds int `json:"stop_timeout_seconds" mapstructure:"stop_timeout_seconds"`
3740
// PoolSize: goroutine pool size for concurrent operations; 0 = runtime.NumCPU().
3841
PoolSize int `json:"pool_size" mapstructure:"pool_size"`
42+
// PullConns: concurrent HTTP Range connections per cloud-image download; <=0 = 8.
43+
PullConns int `json:"pull_conns" mapstructure:"pull_conns"`
3944
// CNIConfDir: CNI plugin configuration dir. Default: /etc/cni/net.d.
4045
CNIConfDir string `json:"cni_conf_dir" mapstructure:"cni_conf_dir"`
4146
// CNIBinDir: CNI plugin binary dir. Default: /opt/cni/bin.
@@ -64,6 +69,11 @@ func (c *Config) EffectivePoolSize() int {
6469
return utils.PoolSizeOrDefault(c.PoolSize)
6570
}
6671

72+
// EffectivePullConns returns PullConns if set, otherwise defaultPullConns.
73+
func (c *Config) EffectivePullConns() int {
74+
return utils.OrDefault(c.PullConns, defaultPullConns)
75+
}
76+
6777
// Validate checks that all config fields are within acceptable ranges.
6878
// Should be called once at startup after unmarshalling.
6979
func (c *Config) Validate() error {

images/cloudimg/cloudimg.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,14 @@ type CloudImg struct {
2929
ops images.Ops[imageIndex, imageEntry]
3030
}
3131

32-
func New(ctx context.Context, rootDir string) (*CloudImg, error) {
33-
cfg := NewConfig(rootDir)
32+
// New builds the cloud image backend under rootDir; pullConns <= 0 defaults to 8 concurrent Range connections.
33+
func New(ctx context.Context, rootDir string, pullConns int) (*CloudImg, error) {
34+
cfg := NewConfig(rootDir, pullConns)
3435
if err := cfg.EnsureDirs(); err != nil {
3536
return nil, fmt.Errorf("ensure dirs: %w", err)
3637
}
3738

38-
log.WithFunc("cloudimg.New").Debug(ctx, "cloud image backend initialized")
39+
log.WithFunc("cloudimg.New").Debugf(ctx, "cloud image backend initialized, pull conns: %d", cfg.PullConns)
3940

4041
store, locker := images.NewStore[imageIndex](cfg.IndexFile(), cfg.IndexLock())
4142
c := &CloudImg{

images/cloudimg/config.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,23 @@ import (
44
"path/filepath"
55

66
"github.com/cocoonstack/cocoon/images"
7+
"github.com/cocoonstack/cocoon/utils"
78
)
89

10+
// defaultPullConns is the default concurrent Range connections per cloud-image download.
11+
const defaultPullConns = 8
12+
913
type Config struct {
1014
images.BaseConfig
15+
// PullConns bounds concurrent HTTP Range connections per cloud-image download.
16+
PullConns int
1117
}
1218

13-
func NewConfig(rootDir string) *Config {
14-
return &Config{BaseConfig: images.BaseConfig{
15-
RootDir: rootDir, Subdir: "cloudimg", BlobExt: ".qcow2",
16-
}}
19+
func NewConfig(rootDir string, pullConns int) *Config {
20+
return &Config{
21+
BaseConfig: images.BaseConfig{RootDir: rootDir, Subdir: "cloudimg", BlobExt: ".qcow2"},
22+
PullConns: utils.OrDefault(pullConns, defaultPullConns),
23+
}
1724
}
1825

1926
func (c *Config) EnsureDirs() error {

images/cloudimg/pull.go

Lines changed: 200 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ import (
88
"io"
99
"net/http"
1010
"os"
11+
"strconv"
12+
"strings"
13+
"sync"
1114
"time"
1215

1316
"github.com/projecteru2/core/log"
@@ -29,29 +32,53 @@ const (
2932
progressInterval = 1 << 20
3033
)
3134

32-
// progressWriter wraps an io.Writer and periodically emits download progress events.
33-
type progressWriter struct {
34-
w io.Writer
35+
// progressCounter emits PhaseDownload events every ~1 MiB; mutex-guarded so it
36+
// serves both the serial writer and parallel range workers.
37+
type progressCounter struct {
38+
mu sync.Mutex
3539
written int64
40+
lastReport int64
3641
total int64
3742
tracker progress.Tracker
38-
lastReport int64
3943
}
4044

41-
func (pw *progressWriter) Write(p []byte) (int, error) {
42-
n, err := pw.w.Write(p)
43-
pw.written += int64(n)
44-
if pw.written-pw.lastReport >= progressInterval {
45-
pw.lastReport = pw.written
46-
pw.tracker.OnEvent(cloudimgProgress.Event{
45+
func (pc *progressCounter) add(n int64) {
46+
pc.mu.Lock()
47+
pc.written += n
48+
report := pc.written-pc.lastReport >= progressInterval
49+
if report {
50+
pc.lastReport = pc.written
51+
}
52+
done := pc.written
53+
pc.mu.Unlock()
54+
// Emit outside the lock: the tracker callback is user-supplied and must not
55+
// serialize the range workers.
56+
if report {
57+
pc.tracker.OnEvent(cloudimgProgress.Event{
4758
Phase: cloudimgProgress.PhaseDownload,
48-
BytesTotal: pw.total,
49-
BytesDone: pw.written,
59+
BytesTotal: pc.total,
60+
BytesDone: done,
5061
})
5162
}
63+
}
64+
65+
// countingWriter forwards writes to w while reporting byte counts to a shared progressCounter.
66+
type countingWriter struct {
67+
w io.Writer
68+
pc *progressCounter
69+
}
70+
71+
func (cw countingWriter) Write(p []byte) (int, error) {
72+
n, err := cw.w.Write(p)
73+
cw.pc.add(int64(n))
5274
return n, err
5375
}
5476

77+
// byteRange is an inclusive [start, end] byte span for an HTTP Range request.
78+
type byteRange struct {
79+
start, end int64
80+
}
81+
5582
// pull commits url as a blob; the URL→blob mapping is idempotent (no-op when the blob already exists).
5683
func pull(ctx context.Context, conf *Config, store storage.Store[imageIndex], url string, force bool, tracker progress.Tracker) error {
5784
logger := log.WithFunc("cloudimg.pull")
@@ -104,20 +131,46 @@ func withDownload(
104131
defer os.Remove(tmpPath) //nolint:errcheck,gosec
105132
defer tmpFile.Close() //nolint:errcheck,gosec
106133

107-
digestHex, err := downloadToFile(ctx, url, tmpFile, tracker)
134+
digestHex, err := downloadToFile(ctx, url, tmpFile, tracker, conf.PullConns)
108135
if err != nil {
109136
return err
110137
}
111138
return fn(tmpFile, tmpPath, digestHex)
112139
}
113140

114-
func downloadToFile(ctx context.Context, url string, dst *os.File, tracker progress.Tracker) (string, error) {
141+
// downloadToFile probes whether url supports HTTP Range requests and downloads it into dst
142+
// accordingly: pullConns concurrent range connections when supported, a single stream otherwise.
143+
func downloadToFile(ctx context.Context, url string, dst *os.File, tracker progress.Tracker, pullConns int) (string, error) {
144+
logger := log.WithFunc("cloudimg.downloadToFile")
145+
client := &http.Client{Timeout: urlDownloadTimeout}
146+
147+
rangeURL, size, ok, err := probeRangeSupport(ctx, client, url)
148+
if err != nil {
149+
return "", err
150+
}
151+
if !ok {
152+
logger.Debugf(ctx, "range requests not supported for %s, falling back to serial download", url)
153+
return downloadSerial(ctx, client, url, dst, tracker)
154+
}
155+
if size > maxDownloadBytes {
156+
return "", fmt.Errorf("download %s: exceeded max size (%d bytes)", url, maxDownloadBytes)
157+
}
158+
159+
logger.Debugf(ctx, "downloading %s in %d parallel range(s)", url, pullConns)
160+
if err := downloadRangesParallel(ctx, client, rangeURL, dst, size, pullConns, tracker); err != nil {
161+
return "", err
162+
}
163+
return hashDigest(dst)
164+
}
165+
166+
// downloadSerial streams url into dst over a single connection; used when the server doesn't
167+
// support Range requests or doesn't report a usable size.
168+
func downloadSerial(ctx context.Context, client *http.Client, url string, dst *os.File, tracker progress.Tracker) (string, error) {
115169
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
116170
if err != nil {
117171
return "", fmt.Errorf("create http request: %w", err)
118172
}
119173

120-
client := &http.Client{Timeout: urlDownloadTimeout}
121174
resp, err := client.Do(req)
122175
if err != nil {
123176
return "", fmt.Errorf("http get %s: %w", url, err)
@@ -138,7 +191,7 @@ func downloadToFile(ctx context.Context, url string, dst *os.File, tracker progr
138191
limitedBody := io.LimitReader(resp.Body, maxDownloadBytes+1)
139192
reader := io.TeeReader(limitedBody, h)
140193

141-
pw := &progressWriter{w: dst, total: contentLength, tracker: tracker}
194+
pw := countingWriter{w: dst, pc: &progressCounter{total: contentLength, tracker: tracker}}
142195
written, err := io.Copy(pw, reader)
143196
if err != nil {
144197
return "", fmt.Errorf("download %s: %w", url, err)
@@ -153,3 +206,134 @@ func downloadToFile(ctx context.Context, url string, dst *os.File, tracker progr
153206

154207
return hex.EncodeToString(h.Sum(nil)), nil
155208
}
209+
210+
// probeRangeSupport issues a GET with Range: bytes=0-0 and reports whether the server serves partial
211+
// content with a known total size. It returns resp.Request.URL (post-redirect) so follow-up range
212+
// requests hit the resolved location directly, since Go's client can drop headers across a
213+
// cross-host redirect on a freshly built request.
214+
func probeRangeSupport(ctx context.Context, client *http.Client, url string) (finalURL string, size int64, ok bool, err error) {
215+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
216+
if err != nil {
217+
return "", 0, false, fmt.Errorf("create range probe request: %w", err)
218+
}
219+
req.Header.Set("Range", "bytes=0-0")
220+
221+
resp, err := client.Do(req)
222+
if err != nil {
223+
return "", 0, false, fmt.Errorf("probe range support for %s: %w", url, err)
224+
}
225+
defer resp.Body.Close() //nolint:errcheck
226+
227+
if resp.StatusCode != http.StatusPartialContent {
228+
return "", 0, false, nil
229+
}
230+
size, ok = parseContentRangeSize(resp.Header.Get("Content-Range"))
231+
if !ok {
232+
return "", 0, false, nil
233+
}
234+
return resp.Request.URL.String(), size, true, nil
235+
}
236+
237+
// downloadRangesParallel splits [0,size) into pullConns contiguous ranges and downloads them
238+
// concurrently into disjoint offsets of dst.
239+
func downloadRangesParallel(ctx context.Context, client *http.Client, url string, dst *os.File, size int64, pullConns int, tracker progress.Tracker) error {
240+
if err := dst.Truncate(size); err != nil {
241+
return fmt.Errorf("truncate temp file: %w", err)
242+
}
243+
244+
tracker.OnEvent(cloudimgProgress.Event{Phase: cloudimgProgress.PhaseDownload, BytesTotal: size})
245+
pc := &progressCounter{total: size, tracker: tracker}
246+
247+
ranges := splitRanges(size, pullConns)
248+
if _, err := utils.Map(ctx, ranges, func(ctx context.Context, _ int, r byteRange) (struct{}, error) {
249+
return struct{}{}, downloadRange(ctx, client, url, dst, r, pc)
250+
}, pullConns); err != nil {
251+
return fmt.Errorf("download %s: %w", url, err)
252+
}
253+
254+
if err := dst.Sync(); err != nil {
255+
return fmt.Errorf("sync temp file: %w", err)
256+
}
257+
258+
tracker.OnEvent(cloudimgProgress.Event{Phase: cloudimgProgress.PhaseDownload, BytesTotal: size, BytesDone: size})
259+
return nil
260+
}
261+
262+
func downloadRange(ctx context.Context, client *http.Client, url string, dst *os.File, r byteRange, pc *progressCounter) error {
263+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
264+
if err != nil {
265+
return fmt.Errorf("create range request: %w", err)
266+
}
267+
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, r.end))
268+
269+
resp, err := client.Do(req)
270+
if err != nil {
271+
return fmt.Errorf("http get range %d-%d: %w", r.start, r.end, err)
272+
}
273+
defer resp.Body.Close() //nolint:errcheck
274+
275+
if resp.StatusCode != http.StatusPartialContent {
276+
return fmt.Errorf("http get range %d-%d: status %s", r.start, r.end, resp.Status)
277+
}
278+
// A mismatched span would land bytes at the wrong offsets; a short body would
279+
// leave a zero hole that hashDigest then blesses with a "valid" digest.
280+
if cr := resp.Header.Get("Content-Range"); !strings.HasPrefix(cr, fmt.Sprintf("bytes %d-%d/", r.start, r.end)) {
281+
return fmt.Errorf("http get range %d-%d: mismatched content-range %q", r.start, r.end, cr)
282+
}
283+
284+
want := r.end - r.start + 1
285+
w := countingWriter{w: io.NewOffsetWriter(dst, r.start), pc: pc}
286+
n, err := io.Copy(w, io.LimitReader(resp.Body, want))
287+
if err != nil {
288+
return fmt.Errorf("write range %d-%d: %w", r.start, r.end, err)
289+
}
290+
if n != want {
291+
return fmt.Errorf("http get range %d-%d: short body %d of %d bytes", r.start, r.end, n, want)
292+
}
293+
return nil
294+
}
295+
296+
// hashDigest re-reads dst from the start and returns its sha256 hex digest. The parallel path can't
297+
// feed a streaming TeeReader since writes land at scattered offsets, so this also verifies what
298+
// actually landed on disk.
299+
func hashDigest(dst *os.File) (string, error) {
300+
if _, err := dst.Seek(0, io.SeekStart); err != nil {
301+
return "", fmt.Errorf("seek temp file: %w", err)
302+
}
303+
h := sha256.New()
304+
if _, err := io.Copy(h, dst); err != nil {
305+
return "", fmt.Errorf("hash temp file: %w", err)
306+
}
307+
return hex.EncodeToString(h.Sum(nil)), nil
308+
}
309+
310+
// splitRanges divides [0,size) into up to n contiguous, inclusive-ended byte ranges.
311+
func splitRanges(size int64, n int) []byteRange {
312+
chunk := (size + int64(n) - 1) / int64(n)
313+
ranges := make([]byteRange, 0, n)
314+
for start := int64(0); start < size; start += chunk {
315+
end := start + chunk - 1
316+
if end >= size {
317+
end = size - 1
318+
}
319+
ranges = append(ranges, byteRange{start: start, end: end})
320+
}
321+
return ranges
322+
}
323+
324+
// parseContentRangeSize extracts the total size from a "Content-Range: bytes 0-0/12345" header value.
325+
func parseContentRangeSize(v string) (int64, bool) {
326+
i := strings.LastIndexByte(v, '/')
327+
if i < 0 || i+1 >= len(v) {
328+
return 0, false
329+
}
330+
total := v[i+1:]
331+
if total == "*" {
332+
return 0, false
333+
}
334+
size, err := strconv.ParseInt(total, 10, 64)
335+
if err != nil || size <= 0 {
336+
return 0, false
337+
}
338+
return size, true
339+
}

0 commit comments

Comments
 (0)