Skip to content

Commit e694dce

Browse files
author
tanq
committed
simple download
1 parent d0ff441 commit e694dce

File tree

6 files changed

+271
-69
lines changed

6 files changed

+271
-69
lines changed

‎cmd/root.go‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ package cmd
22

33
import (
44
"fmt"
5+
u "net/url"
56
"os"
6-
"runtime"
7+
"strings"
78
"time"
89

910
"github.com/rs/zerolog/log"
@@ -12,7 +13,7 @@ import (
1213
)
1314

1415
var (
15-
url string
16+
// url string
1617
output string
1718
connections int
1819
timeout time.Duration
@@ -32,18 +33,25 @@ var rootCmd = &cobra.Command{
3233
internal.InitLogger(debug)
3334
log.Debug().Msg("Debug logging enabled")
3435
},
36+
Args: cobra.ArbitraryArgs,
3537
Run: func(cmd *cobra.Command, args []string) {
36-
if urlListFile != "" && url != "" {
37-
log.Fatal().Msg("Cannot specify both --url and --urllist, choose one")
38+
if len(args) == 0 && urlListFile == "" {
39+
log.Fatal().Msg("No URL or URL list provided")
40+
}
41+
url := args[0]
42+
if _, err := u.Parse(url); err != nil {
43+
log.Fatal().Err(err).Msg("Invalid URL format")
3844
}
39-
if urlListFile == "" && url == "" {
40-
log.Fatal().Msg("Must specify either --url or --urllist")
45+
if urlListFile != "" && url != "" {
46+
log.Fatal().Msg("Cannot specify both argument and --urllist, choose one")
4147
}
4248

4349
// Handle single URL download
4450
if url != "" {
4551
if output == "" {
46-
log.Fatal().Msg("Output file path is required with --url")
52+
parsedURL, _ := u.Parse(url)
53+
output = strings.Split(parsedURL.Path, "/")[len(strings.Split(parsedURL.Path, "/"))-1]
54+
log.Debug().Str("output", output).Msg("Output file path not specified, using URL path")
4755
}
4856
entries := []internal.DownloadEntry{{URL: url, OutputPath: output}}
4957
err := internal.BatchDownload(entries, 1, connections, timeout, kaTimeout, userAgent, proxyURL)
@@ -71,6 +79,27 @@ var rootCmd = &cobra.Command{
7179
},
7280
}
7381

82+
var simpleCmd = &cobra.Command{
83+
Use: "simple",
84+
Short: "Simple mode for single threaded direct download",
85+
Args: cobra.ExactArgs(1),
86+
Run: func(cmd *cobra.Command, args []string) {
87+
url := args[0]
88+
if _, err := u.Parse(url); err != nil {
89+
log.Fatal().Err(err).Msg("Invalid URL format")
90+
}
91+
if output == "" {
92+
parsedURL, _ := u.Parse(url)
93+
output = strings.Split(parsedURL.Path, "/")[len(strings.Split(parsedURL.Path, "/"))-1]
94+
log.Debug().Str("output", output).Msg("Output file path not specified, using URL path")
95+
}
96+
err := internal.SimpleDownload(url, output)
97+
if err != nil {
98+
log.Fatal().Err(err).Msg("Download failed")
99+
}
100+
},
101+
}
102+
74103
var cleanCmd = &cobra.Command{
75104
Use: "clean",
76105
Short: "Clean up temporary files",
@@ -91,17 +120,20 @@ func Execute() {
91120
}
92121

93122
func init() {
94-
rootCmd.Flags().StringVarP(&url, "url", "u", "", "URL to download")
123+
// rootCmd.Flags().StringVarP(&url, "url", "u", "", "URL to download")
95124
rootCmd.Flags().StringVarP(&output, "output", "o", "", "Output file path (required with --url/-u)")
96125
rootCmd.Flags().StringVarP(&urlListFile, "urllist", "l", "", "Path to YAML file containing URLs and output paths")
97126
rootCmd.Flags().IntVarP(&numLinks, "workers", "w", 1, "Number of links to download in parallel (default: 1)")
98-
rootCmd.Flags().IntVarP(&connections, "connections", "c", min(runtime.NumCPU(), 32), "Number of connections per download (default: CPU cores)")
127+
rootCmd.Flags().IntVarP(&connections, "connections", "c", 4, "Number of connections per download (default: 4)")
99128
rootCmd.Flags().DurationVarP(&timeout, "timeout", "t", 3*time.Minute, "Connection timeout (eg., 5s, 10m; default: 3m)")
100129
rootCmd.Flags().DurationVarP(&kaTimeout, "keep-alive-timeout", "k", 90*time.Second, "Keep-alive timeout for client (eg./ 10s, 1m, 80s; default: 90s)")
101-
rootCmd.Flags().StringVarP(&userAgent, "user-agent", "a", "Danzo/1337", "User agent")
130+
rootCmd.Flags().StringVarP(&userAgent, "user-agent", "a", internal.ToolUserAgent, "User agent")
102131
rootCmd.Flags().StringVarP(&proxyURL, "proxy", "p", "", "HTTP/HTTPS proxy URL (e.g., proxy.example.com:8080)")
103132
rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug logging")
104133

105134
rootCmd.AddCommand(cleanCmd)
106135
cleanCmd.Flags().StringVarP(&cleanOutput, "output", "o", "", "Output file path")
136+
137+
rootCmd.AddCommand(simpleCmd)
138+
simpleCmd.Flags().StringVarP(&output, "output", "o", "", "Output file path")
107139
}

‎internal/downloader.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,18 @@ func BatchDownload(entries []DownloadEntry, numLinks int, connectionsPerLink int
5454
doneCh := make(chan struct{})
5555
client := createHTTPClient(config.Timeout, config.KATimeout, config.ProxyURL)
5656
fileSize, err := getFileSize(config.URL, config.UserAgent, client)
57-
if err != nil {
57+
58+
if err == ErrRangeRequestsNotSupported {
59+
// file size unknown, so can't show progress; so track bytes downloaded
60+
logger.Debug().Str("url", entry.URL).Msg("Range requests not supported, using simple download")
61+
progressManager.Register(entry.OutputPath, -1) // -1 means unknown size
62+
} else if err != nil {
5863
logger.Error().Err(err).Str("output", entry.OutputPath).Msg("Failed to get file size")
5964
errorCh <- fmt.Errorf("error getting file size for %s: %v", entry.URL, err)
6065
continue
66+
} else {
67+
progressManager.Register(entry.OutputPath, fileSize)
6168
}
62-
progressManager.Register(entry.OutputPath, fileSize)
6369
var internalWg sync.WaitGroup
6470
internalWg.Add(1)
6571

@@ -83,7 +89,11 @@ func BatchDownload(entries []DownloadEntry, numLinks int, connectionsPerLink int
8389
}
8490
}(entry.OutputPath, progressCh, doneCh)
8591

86-
err = downloadWithProgress(config, fileSize, progressCh)
92+
if err == ErrRangeRequestsNotSupported {
93+
err = performSimpleDownload(entry.URL, entry.OutputPath, client, config.UserAgent, progressCh)
94+
} else {
95+
err = downloadWithProgress(config, fileSize, progressCh)
96+
}
8797
close(doneCh)
8898
internalWg.Wait()
8999

‎internal/helpers.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import (
1616
)
1717

1818
const bufferSize = 1024 * 1024 * 2 // 2MB buffer
19+
const ToolUserAgent = "danzo/1337"
20+
1921
var chunkIDRegex = regexp.MustCompile(`\.part(\d+)$`)
2022

2123
type DownloadConfig struct {

‎internal/operations.go‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
"time"
1414
)
1515

16+
var ErrRangeRequestsNotSupported = errors.New("server doesn't support range requests")
17+
1618
func getFileSize(url string, userAgent string, client *http.Client) (int64, error) {
1719
log := GetLogger("filesize")
1820
req, err := http.NewRequest("HEAD", url, nil)
@@ -27,7 +29,8 @@ func getFileSize(url string, userAgent string, client *http.Client) (int64, erro
2729
}
2830
defer resp.Body.Close()
2931
if resp.Header.Get("Accept-Ranges") != "bytes" {
30-
return 0, errors.New("server doesn't support range requests")
32+
log.Warn().Msg("Server doesn't support range requests, will use simple download")
33+
return 0, ErrRangeRequestsNotSupported
3134
}
3235
contentLength := resp.Header.Get("Content-Length")
3336
if contentLength == "" {
@@ -44,6 +47,49 @@ func getFileSize(url string, userAgent string, client *http.Client) (int64, erro
4447
return size, nil
4548
}
4649

50+
func parseContentLength(contentLength string) (int64, error) {
51+
var size int64
52+
_, err := fmt.Sscanf(contentLength, "%d", &size)
53+
if err != nil {
54+
return -1, fmt.Errorf("invalid content length: %v", err)
55+
}
56+
if size <= 0 {
57+
return -1, fmt.Errorf("invalid file size reported by server")
58+
}
59+
return size, nil
60+
}
61+
62+
// func getFileSize(url string, userAgent string, client *http.Client) (int64, error) {
63+
// log := GetLogger("filesize")
64+
// req, err := http.NewRequest("HEAD", url, nil)
65+
// if err != nil {
66+
// return 0, err
67+
// }
68+
// req.Header.Set("User-Agent", userAgent)
69+
// log.Debug().Str("url", url).Msg("Sending HEAD request")
70+
// resp, err := client.Do(req)
71+
// if err != nil {
72+
// return 0, err
73+
// }
74+
// defer resp.Body.Close()
75+
// if resp.Header.Get("Accept-Ranges") != "bytes" {
76+
// return 0, errors.New("server doesn't support range requests")
77+
// }
78+
// contentLength := resp.Header.Get("Content-Length")
79+
// if contentLength == "" {
80+
// return 0, errors.New("server didn't provide Content-Length header")
81+
// }
82+
// size, err := strconv.ParseInt(contentLength, 10, 64)
83+
// if err != nil {
84+
// return 0, fmt.Errorf("invalid content length: %v", err)
85+
// }
86+
// if size <= 0 {
87+
// return 0, errors.New("invalid file size reported by server")
88+
// }
89+
// log.Debug().Int64("bytes", size).Msg("File size determined")
90+
// return size, nil
91+
// }
92+
4793
func downloadChunk(job *DownloadJob, chunk *DownloadChunk, client *http.Client, wg *sync.WaitGroup, progressCh chan<- int64, mutex *sync.Mutex) {
4894
log := GetLogger("chunk").With().Int("chunkId", chunk.ID).Logger()
4995
defer wg.Done()

‎internal/process-manager.go‎ renamed to ‎internal/progress-manager.go‎

Lines changed: 60 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ func (pm *ProgressManager) Update(outputPath string, bytesDownloaded int64) {
5454
func (pm *ProgressManager) Complete(outputPath string, totalDownloaded int64) {
5555
pm.mutex.Lock()
5656
defer pm.mutex.Unlock()
57+
log.Debug().Msg("REACHED HERE - Complete")
5758
if info, exists := pm.progressMap[outputPath]; exists {
5859
info.Completed = true
5960
info.CompletedSize = totalDownloaded
@@ -84,66 +85,70 @@ func (pm *ProgressManager) StartDisplay() {
8485
fmt.Printf("\r\033[K") // Clear the current line
8586
}
8687

87-
for {
88-
select {
89-
case <-ticker.C:
90-
pm.mutex.RLock()
91-
activePaths := map[int]string{}
92-
innerIndex := 0
93-
for path, info := range pm.progressMap {
94-
if !info.Completed {
95-
activePaths[innerIndex] = path
96-
innerIndex++
97-
}
88+
displayProgress := func() {
89+
pm.mutex.RLock()
90+
activePaths := map[int]string{}
91+
innerIndex := 0
92+
for path, info := range pm.progressMap {
93+
if !info.Completed {
94+
activePaths[innerIndex] = path
95+
innerIndex++
9896
}
99-
if len(activePaths) > 0 {
100-
if currentIndex >= len(activePaths) {
101-
currentIndex = 0
102-
}
103-
outputPath := activePaths[currentIndex]
104-
info := pm.progressMap[outputPath]
105-
now := time.Now()
106-
lastTime, exists := lastUpdateTimes[outputPath]
107-
if !exists {
108-
lastTime = info.StartTime
109-
lastDownloaded[outputPath] = 0
110-
}
111-
timeDiff := now.Sub(lastTime).Seconds()
112-
byteDiff := info.Downloaded - lastDownloaded[outputPath]
113-
if timeDiff > 0 {
114-
info.Speed = float64(byteDiff) / timeDiff / 1024 / 1024 // MB/s
115-
lastUpdateTimes[outputPath] = now
116-
lastDownloaded[outputPath] = info.Downloaded
117-
}
118-
elapsed := time.Since(info.StartTime).Seconds()
119-
if elapsed > 0 {
120-
info.AvgSpeed = float64(info.Downloaded) / elapsed / 1024 / 1024 // MB/s
121-
}
122-
if info.Speed > 0 {
123-
etaSeconds := int64(float64(info.TotalSize-info.Downloaded) / (info.Speed * 1024 * 1024)) // from MB/s to B/s
124-
if etaSeconds < 60 {
125-
info.ETA = fmt.Sprintf("%ds", etaSeconds)
126-
} else if etaSeconds < 3600 {
127-
info.ETA = fmt.Sprintf("%dm %ds", etaSeconds/60, etaSeconds%60)
128-
} else {
129-
info.ETA = fmt.Sprintf("%dh %dm", etaSeconds/3600, (etaSeconds%3600)/60)
130-
}
97+
}
98+
if len(activePaths) > 0 {
99+
if currentIndex >= len(activePaths) {
100+
currentIndex = 0
101+
}
102+
outputPath := activePaths[currentIndex]
103+
info := pm.progressMap[outputPath]
104+
now := time.Now()
105+
lastTime, exists := lastUpdateTimes[outputPath]
106+
if !exists {
107+
lastTime = info.StartTime
108+
lastDownloaded[outputPath] = 0
109+
}
110+
timeDiff := now.Sub(lastTime).Seconds()
111+
byteDiff := info.Downloaded - lastDownloaded[outputPath]
112+
if timeDiff > 0 {
113+
info.Speed = float64(byteDiff) / timeDiff / 1024 / 1024 // MB/s
114+
lastUpdateTimes[outputPath] = now
115+
lastDownloaded[outputPath] = info.Downloaded
116+
}
117+
elapsed := time.Since(info.StartTime).Seconds()
118+
if elapsed > 0 {
119+
info.AvgSpeed = float64(info.Downloaded) / elapsed / 1024 / 1024 // MB/s
120+
}
121+
if info.Speed > 0 {
122+
etaSeconds := int64(float64(info.TotalSize-info.Downloaded) / (info.Speed * 1024 * 1024)) // from MB/s to B/s
123+
if etaSeconds < 60 {
124+
info.ETA = fmt.Sprintf("%ds", etaSeconds)
125+
} else if etaSeconds < 3600 {
126+
info.ETA = fmt.Sprintf("%dm %ds", etaSeconds/60, etaSeconds%60)
131127
} else {
132-
info.ETA = "calculating..."
128+
info.ETA = fmt.Sprintf("%dh %dm", etaSeconds/3600, (etaSeconds%3600)/60)
133129
}
134-
percent := float64(info.Downloaded) / float64(info.TotalSize) * 100
135-
136-
// Display progress
137-
clearLine()
138-
fmt.Printf("[%s] %.2f%% (%s/%s) Speed: %.2f MB/s ETA: %s", outputPath, percent, formatBytes(uint64(info.Downloaded)), formatBytes(uint64(info.TotalSize)), info.Speed, info.ETA)
139-
log.Debug().Str("file", outputPath).Float64("percent", percent).Str("downloaded", formatBytes(uint64(info.Downloaded))).Str("total", formatBytes(uint64(info.TotalSize))).Float64("speed_mbps", info.Speed).Str("eta", info.ETA).Msg("Download progress")
140-
} else if pm.IsAllCompleted() {
141-
clearLine()
142-
fmt.Print("Performing assemble or waiting for a job...")
130+
} else {
131+
info.ETA = "calculating..."
143132
}
144-
currentIndex++
145-
pm.mutex.RUnlock()
133+
percent := float64(info.Downloaded) / float64(info.TotalSize) * 100
146134

135+
// Display progress
136+
clearLine()
137+
fmt.Printf("[%s] %.2f%% (%s/%s) Speed: %.2f MB/s ETA: %s", outputPath, percent, formatBytes(uint64(info.Downloaded)), formatBytes(uint64(info.TotalSize)), info.Speed, info.ETA)
138+
log.Debug().Str("file", outputPath).Float64("percent", percent).Str("downloaded", formatBytes(uint64(info.Downloaded))).Str("total", formatBytes(uint64(info.TotalSize))).Float64("speed_mbps", info.Speed).Str("eta", info.ETA).Msg("Download progress")
139+
} else if pm.IsAllCompleted() {
140+
clearLine()
141+
fmt.Print("Performing assemble or waiting for a job...")
142+
}
143+
currentIndex++
144+
pm.mutex.RUnlock()
145+
}
146+
147+
displayProgress()
148+
for {
149+
select {
150+
case <-ticker.C:
151+
displayProgress()
147152
case <-pm.doneCh:
148153
clearLine()
149154
return

0 commit comments

Comments
 (0)