Feat/multi thread download artifacts (#827)

* feat: support to download job artifacts in multiple threads

* support to set the thread num

* use release of http-downloader

---------

Co-authored-by: rick <[email protected]>

Author: LinuxSuRen

app/cmd/job_artifact_download.go

@@ -4,16 +4,17 @@ import (
 	"fmt"
 	appCfg "github.com/jenkins-zh/jenkins-cli/app/config"
 	"github.com/jenkins-zh/jenkins-cli/app/i18n"
+	httpdownloader "github.com/linuxsuren/http-downloader/pkg/net"
 	"net/http"
 	"net/url"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 
 	"github.com/jenkins-zh/jenkins-cli/app/helper"
 
 	"github.com/jenkins-zh/jenkins-cli/client"
-	httpdownloader "github.com/linuxsuren/http-downloader/pkg"
 	"github.com/spf13/cobra"
 )
 
@@ -23,6 +24,7 @@ type JobArtifactDownloadOption struct {
 	ShowProgress bool
 	DownloadDir  string
 
+	Thread       int
 	Jenkins      *appCfg.JenkinsServer
 	RoundTripper http.RoundTripper
 }
@@ -37,6 +39,8 @@ func init() {
 		i18n.T("Whether show the progress"))
 	jobArtifactDownloadCmd.Flags().StringVarP(&jobArtifactDownloadOption.DownloadDir, "download-dir", "", "",
 		i18n.T("The directory which artifact will be downloaded"))
+	jobArtifactDownloadCmd.Flags().IntVarP(&jobArtifactDownloadOption.Thread, "thread", "t", runtime.NumCPU(),
+		i18n.T("The number of threads"))
 }
 
 var jobArtifactDownloadCmd = &cobra.Command{
@@ -86,16 +90,12 @@ func (j *JobArtifactDownloadOption) download(artifactURL, fileName string) (err
 	jenkinsURL, _ := url.Parse(j.Jenkins.URL)
 	targetURL := fmt.Sprintf("%s%s", j.Jenkins.URL, strings.TrimPrefix(artifactURL, jenkinsURL.Path))
 	fmt.Println("start to download from", targetURL)
-	downloader := httpdownloader.HTTPDownloader{
-		RoundTripper:   j.RoundTripper,
-		TargetFilePath: fileName,
-		URL:            targetURL,
-		UserName:       j.Jenkins.UserName,
-		Password:       j.Jenkins.Token,
-		Proxy:          j.Jenkins.Proxy,
-		ProxyAuth:      j.Jenkins.ProxyAuth,
-		ShowProgress:   j.ShowProgress,
-	}
-	err = downloader.DownloadFile()
+
+	download := &httpdownloader.MultiThreadDownloader{}
+	download.WithBasicAuth(j.Jenkins.UserName, j.Jenkins.Token)
+	download.WithShowProgress(j.ShowProgress)
+	download.WithKeepParts(true)
+	download.WithRoundTripper(j.RoundTripper)
+	err = download.Download(targetURL, fileName, j.Thread)
 	return
 }

go.mod

@@ -20,44 +20,47 @@ require (
 	github.com/linuxsuren/cobra-extension v0.0.16
 	github.com/linuxsuren/go-cli-alias v0.0.10
 	github.com/linuxsuren/go-cli-plugin v0.0.5
-	github.com/linuxsuren/http-downloader v0.0.85
+	github.com/linuxsuren/http-downloader v0.0.98
 	github.com/magiconair/properties v1.8.5
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/moby v20.10.8+incompatible
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.18.1
+	github.com/onsi/gomega v1.27.10
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.4
 	github.com/zalando/go-keyring v0.0.0-20200121091418-667557018717
 	go.uber.org/zap v1.19.0
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
+	golang.org/x/net v0.25.0
 	golang.org/x/text v0.21.0
 	gopkg.in/yaml.v2 v2.4.0
 	moul.io/http2curl v1.0.0
 )
 
 require (
-	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/containerd/containerd v1.5.5 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/creack/pty v1.1.17 // indirect
 	github.com/danieljoos/wincred v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/gliderlabs/ssh v0.3.5 // indirect
 	github.com/godbus/dbus v4.1.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
@@ -71,23 +74,28 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/schollz/progressbar/v3 v3.13.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/src-d/gcfg v1.4.0 // indirect
-	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
-	google.golang.org/grpc v1.33.2 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect
+	google.golang.org/grpc v1.40.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect