configurable keep alive t/o; randomize user agents

tanq · tanq · commit c35861466bf7 · 2025-03-14T13:59:07.000-04:00
diff --git a/README.md b/README.md
@@ -53,25 +53,31 @@ go build .
 ### Command Options
 
 ```
+Danzo is a fast CLI download manager
+
 Usage:
   danzo [flags]
   danzo [command]
 
 Available Commands:
   clean       Clean up temporary files
+  completion  Generate the autocompletion script for the specified shell
   help        Help about any command
 
 Flags:
-  -c, --connections int        Number of connections per download (default: # CPU cores)
-      --debug                  Enable debug logging
-  -h, --help                   help for danzo
-  -o, --output string          Output file path (required with --url/-u)
-  -p, --proxy string           HTTP/HTTPS proxy URL (e.g., proxy.example.com:8080)
-  -t, --timeout duration       Connection timeout (default 3m0s)
-  -u, --url string             URL to download
-  -a, --user-agent string      User agent (default "Danzo/1337")
-  -l, --urllist string         Path to YAML file containing URLs and output paths
-  -w, --workers int            Number of links to download in parallel (default 1)
+  -c, --connections int               Number of connections per download (default: CPU cores) (default 16)
+      --debug                         Enable debug logging
+  -h, --help                          help for danzo
+  -k, --keep-alive-timeout duration   Keep-alive timeout for client (eg./ 10s, 1m, 80s; default: 90s) (default 1m30s)
+  -o, --output string                 Output file path (required with --url/-u)
+  -p, --proxy string                  HTTP/HTTPS proxy URL (e.g., proxy.example.com:8080)
+  -t, --timeout duration              Connection timeout (eg., 5s, 10m; default: 3m) (default 3m0s)
+  -u, --url string                    URL to download
+  -l, --urllist string                Path to YAML file containing URLs and output paths
+  -a, --user-agent string             User agent (default "Danzo/1337")
+  -w, --workers int                   Number of links to download in parallel (default: 1) (default 1)
+
+Use "danzo [command] --help" for more information about a command.
 ```
 
 ### Batch Download
@@ -118,3 +124,4 @@ Danzo stores partial downloads on disk in the `.danzo-temp` directory (situated
 - For servers with rate limiting, reducing the number of connections might help
 - Debug mode (`--debug`) provides detailed information about the download process
 - Temporary files are stored in a .danzo-temp directory and automatically cleaned up after download
+- Use `-a randomize` to randomize the user agent for every HTTP client
diff --git a/cmd/root.go b/cmd/root.go
@@ -16,6 +16,7 @@ var (
 	output      string
 	connections int
 	timeout     time.Duration
+	kaTimeout   time.Duration
 	userAgent   string
 	proxyURL    string
 	debug       bool
@@ -45,7 +46,7 @@ var rootCmd = &cobra.Command{
 				log.Fatal().Msg("Output file path is required with --url")
 			}
 			entries := []internal.DownloadEntry{{URL: url, OutputPath: output}}
-			err := internal.BatchDownload(entries, 1, connections, timeout, userAgent, proxyURL)
+			err := internal.BatchDownload(entries, 1, connections, timeout, kaTimeout, userAgent, proxyURL)
 			if err != nil {
 				log.Fatal().Err(err).Msg("Download failed")
 			}
@@ -63,7 +64,7 @@ var rootCmd = &cobra.Command{
 			connectionsPerLink = max(maxConnections/numLinks, 1)
 			log.Warn().Int("connections", connectionsPerLink).Int("numLinks", numLinks).Msg("adjusted connections to below max limit")
 		}
-		err = internal.BatchDownload(entries, numLinks, connectionsPerLink, timeout, userAgent, proxyURL)
+		err = internal.BatchDownload(entries, numLinks, connectionsPerLink, timeout, kaTimeout, userAgent, proxyURL)
 		if err != nil {
 			log.Fatal().Err(err).Msg("Batch download completed with errors")
 		}
@@ -96,6 +97,7 @@ func init() {
 	rootCmd.Flags().IntVarP(&numLinks, "workers", "w", 1, "Number of links to download in parallel (default: 1)")
 	rootCmd.Flags().IntVarP(&connections, "connections", "c", min(runtime.NumCPU(), 32), "Number of connections per download (default: CPU cores)")
 	rootCmd.Flags().DurationVarP(&timeout, "timeout", "t", 3*time.Minute, "Connection timeout (eg., 5s, 10m; default: 3m)")
+	rootCmd.Flags().DurationVarP(&kaTimeout, "keep-alive-timeout", "k", 90*time.Second, "Keep-alive timeout for client (eg./ 10s, 1m, 80s; default: 90s)")
 	rootCmd.Flags().StringVarP(&userAgent, "user-agent", "a", "Danzo/1337", "User agent")
 	rootCmd.Flags().StringVarP(&proxyURL, "proxy", "p", "", "HTTP/HTTPS proxy URL (e.g., proxy.example.com:8080)")
 	rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug logging")
diff --git a/internal/downloader.go b/internal/downloader.go
@@ -8,7 +8,7 @@ import (
 	"time"
 )
 
-func BatchDownload(entries []DownloadEntry, numLinks int, connectionsPerLink int, timeout time.Duration, userAgent string, proxyURL string) error {
+func BatchDownload(entries []DownloadEntry, numLinks int, connectionsPerLink int, timeout time.Duration, kaTimeout time.Duration, userAgent string, proxyURL string) error {
 	log := GetLogger("downloader")
 	log.Info().Int("totalFiles", len(entries)).Int("numLinks", numLinks).Int("connections", connectionsPerLink).Msg("Starting batch download")
 
@@ -38,17 +38,21 @@ func BatchDownload(entries []DownloadEntry, numLinks int, connectionsPerLink int
 			logger := log.With().Int("workerID", workerID).Logger()
 			for entry := range entriesCh {
 				logger.Debug().Str("output", entry.OutputPath).Msg("Worker starting download")
+				if userAgent == "randomize" {
+					userAgent = getRandomUserAgent()
+				}
 				config := DownloadConfig{
 					URL:         entry.URL,
 					OutputPath:  entry.OutputPath,
 					Connections: connectionsPerLink,
 					Timeout:     timeout,
+					KATimeout:   kaTimeout,
 					UserAgent:   userAgent,
 					ProxyURL:    proxyURL,
 				}
 				progressCh := make(chan int64)
 				doneCh := make(chan struct{})
-				client := createHTTPClient(config.Timeout, config.ProxyURL)
+				client := createHTTPClient(config.Timeout, config.KATimeout, config.ProxyURL)
 				fileSize, err := getFileSize(config.URL, config.UserAgent, client)
 				if err != nil {
 					logger.Error().Err(err).Str("output", entry.OutputPath).Msg("Failed to get file size")
@@ -108,7 +112,7 @@ func BatchDownload(entries []DownloadEntry, numLinks int, connectionsPerLink int
 
 func downloadWithProgress(config DownloadConfig, fileSize int64, progressCh chan<- int64) error {
 	log := GetLogger("download-worker")
-	client := createHTTPClient(config.Timeout, config.ProxyURL)
+	client := createHTTPClient(config.Timeout, config.KATimeout, config.ProxyURL)
 	log.Debug().Str("size", formatBytes(uint64(fileSize))).Msg("File size determined")
 	job := DownloadJob{
 		Config:    config,
diff --git a/internal/helpers.go b/internal/helpers.go
@@ -23,6 +23,7 @@ type DownloadConfig struct {
 	OutputPath  string
 	Connections int
 	Timeout     time.Duration
+	KATimeout   time.Duration
 	ProxyURL    string
 	UserAgent   string
 }
@@ -53,6 +54,35 @@ type DownloadEntry struct {
 	URL        string `yaml:"link"`
 }
 
+func getRandomUserAgent() string {
+	userAgents := []string{
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15",
+		"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 YaBrowser/27.7.7.7 Yowser/2.5 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0",
+		"curl/7.88.1",
+		"Wget/1.21.4",
+	}
+	return userAgents[time.Now().UnixNano()%int64(len(userAgents))]
+}
+
 func ReadDownloadList(filePath string) ([]DownloadEntry, error) {
 	log := GetLogger("config")
 	data, err := os.ReadFile(filePath)
@@ -76,11 +106,11 @@ func ReadDownloadList(filePath string) ([]DownloadEntry, error) {
 	return entries, nil
 }
 
-func createHTTPClient(timeout time.Duration, proxyURL string) *http.Client {
+func createHTTPClient(timeout time.Duration, keepAliveTO time.Duration, proxyURL string) *http.Client {
 	transport := &http.Transport{
 		MaxIdleConns:        100,
 		MaxIdleConnsPerHost: 100, // for connection reuse
-		IdleConnTimeout:     90 * time.Second,
+		IdleConnTimeout:     keepAliveTO,
 		DisableCompression:  true,
 		MaxConnsPerHost:     0,
 		// These two seem to reduce performance drastically with custom dial context
