Merge pull request #4 from Chikage0o0/goroutine

run x downloads parallel in goroutine

cmd/root.go

@@ -1,18 +1,17 @@
 package cmd
 
 import (
-	"github.com/nothub/mrpack-install/http"
 	modrinth "github.com/nothub/mrpack-install/modrinth/api"
 	"github.com/nothub/mrpack-install/modrinth/mrpack"
+	"github.com/nothub/mrpack-install/requester"
 	"github.com/nothub/mrpack-install/server"
+	"github.com/spf13/cobra"
 	"log"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-
-	"github.com/spf13/cobra"
 )
 
 func init() {
@@ -22,6 +21,8 @@ func init() {
 	rootCmd.Flags().String("server-dir", "mc", "Server directory path")
 	rootCmd.Flags().String("server-file", "", "Server jar file name")
 	rootCmd.Flags().String("proxy", "", "Use a proxy to download")
+	rootCmd.Flags().Int("download-thread", 8, "Download threads")
+	rootCmd.Flags().Int("retry-times", 3, "Number of retries when a download fails")
 }
 
 var rootCmd = &cobra.Command{
@@ -41,17 +42,26 @@ var rootCmd = &cobra.Command{
 		if err != nil {
 			log.Fatalln(err)
 		}
-
 		proxy, err := cmd.Flags().GetString("proxy")
 		if err != nil {
 			log.Fatalln(err)
 		}
 		if proxy != "" {
-			err := http.Instance.SetProxy(proxy)
+			err := requester.DefaultHttpClient.SetProxy(proxy)
 			if err != nil {
 				log.Fatalln(err)
 			}
 		}
+		downloadThreads, err := cmd.Flags().GetInt("download-thread")
+		if err != nil || downloadThreads > 64 {
+			downloadThreads = 8
+			log.Println(err)
+		}
+		retryTimes, err := cmd.Flags().GetInt("retry-times")
+		if err != nil {
+			retryTimes = 3
+			log.Println(err)
+		}
 
 		input := args[0]
 		version := ""
@@ -70,7 +80,7 @@ var rootCmd = &cobra.Command{
 
 		} else if isUrl(input) {
 			log.Println("Downloading mrpack file from", args)
-			file, err := http.Instance.DownloadFile(input, serverDir, "")
+			file, err := requester.DefaultHttpClient.DownloadFile(input, serverDir, "")
 			if err != nil {
 				log.Fatalln(err)
 			}
@@ -104,7 +114,7 @@ var rootCmd = &cobra.Command{
 			for i := range files {
 				if strings.HasSuffix(files[i].Filename, ".mrpack") {
 					log.Println("Downloading mrpack file from", files[i].Url)
-					file, err := http.Instance.DownloadFile(files[i].Url, serverDir, "")
+					file, err := requester.DefaultHttpClient.DownloadFile(files[i].Url, serverDir, "")
 					if err != nil {
 						log.Fatalln(err)
 					}
@@ -152,7 +162,7 @@ var rootCmd = &cobra.Command{
 				log.Fatalln(err)
 			}
 			log.Println("Downloading server file from", u)
-			_, err = http.Instance.DownloadFile(u, serverDir, serverFile)
+			_, err = requester.DefaultHttpClient.DownloadFile(u, serverDir, serverFile)
 			if err != nil {
 				log.Fatalln(err)
 			}
@@ -162,32 +172,33 @@ var rootCmd = &cobra.Command{
 
 		// download mods
 		log.Printf("Downloading %v dependencies...\n", len(index.Files))
+		var downloadPoolArray []*requester.DownloadPool
 		for i := range index.Files {
 			file := index.Files[i]
 			if file.Env.Server == modrinth.UnsupportedEnvSupport {
 				continue
 			}
-			success := false
-			// TODO: run x downloads parallel in goroutine
-			for j := range file.Downloads {
-				f, err := http.Instance.DownloadFile(file.Downloads[j], path.Join(serverDir, filepath.Dir(file.Path)), filepath.Base(file.Path))
-				if err != nil {
-					log.Println(err)
-				} else {
-					log.Println("Dependency downloaded:", f)
-				}
-				success = true
-			}
-			if !success {
-				log.Fatalf("Unable to download dependency: %+v\n", file)
-			}
+			downloadPoolArray = append(downloadPoolArray, requester.NewDownloadPool(file.Downloads, map[string]string{"sha1": string(file.Hashes.Sha1)}, filepath.Base(file.Path), path.Join(serverDir, filepath.Dir(file.Path))))
 		}
 
+		downloadPools := requester.NewDownloadPools(requester.DefaultHttpClient, downloadPoolArray, downloadThreads, retryTimes)
+		downloadPools.Do()
 		log.Println("Extracting overrides...")
 		err = mrpack.ExtractOverrides(archivePath, serverDir)
 		if err != nil {
 			log.Fatalln(err)
 		}
+		uncleanNotification := false
+		for i := range downloadPools.DownloadPool {
+			dl := downloadPools.DownloadPool[i]
+			if !dl.Success {
+				uncleanNotification = true
+				log.Println("Dependency downloaded Fail:", dl.FileName)
+			}
+		}
+		if uncleanNotification {
+			log.Fatalln("Download failed,You can fix the error manually")
+		}
 
 		log.Println("Done :) Have a nice day ✌️")
 	},

cmd/server.go

@@ -2,8 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/nothub/mrpack-install/http"
 	"github.com/nothub/mrpack-install/mojang"
+	"github.com/nothub/mrpack-install/requester"
 	"github.com/nothub/mrpack-install/server"
 	"github.com/spf13/cobra"
 	"log"
@@ -95,7 +95,7 @@ var serverCmd = &cobra.Command{
 			log.Fatalln(err)
 		}
 
-		file, err := http.Instance.DownloadFile(url, serverDir, serverFile)
+		file, err := requester.DefaultHttpClient.DownloadFile(url, serverDir, serverFile)
 		if err != nil {
 			return
 		}

modrinth/api/client.go

@@ -1,19 +1,19 @@
 package api
 
 import (
-	"github.com/nothub/mrpack-install/http"
+	"github.com/nothub/mrpack-install/requester"
 	"log"
 	"net/url"
 )
 
 type ModrinthClient struct {
-	Http    *http.Client
+	Http    *requester.HTTPClient
 	BaseUrl string
 }
 
 func NewClient(host string) *ModrinthClient {
 	client := ModrinthClient{
-		Http: http.Instance,
+		Http: requester.DefaultHttpClient,
 	}
 	u, err := url.Parse("https://" + host + "/")
 	if err != nil {

modrinth/mrpack/mrpack_test.go

@@ -1,18 +1,18 @@
 package mrpack
 
 import (
-	"github.com/nothub/mrpack-install/http"
+	"github.com/nothub/mrpack-install/requester"
 	"log"
 	"os"
 	"testing"
 )
 
 func init() {
-	_, err := http.Instance.DownloadFile("https://cdn.modrinth.com/data/1KVo5zza/versions/1vRDfe1u/MR_Fabulously%20Optimized_4.2.1.mrpack", os.TempDir(), "")
+	_, err := requester.DefaultHttpClient.DownloadFile("https://cdn.modrinth.com/data/1KVo5zza/versions/1vRDfe1u/MR_Fabulously%20Optimized_4.2.1.mrpack", os.TempDir(), "")
 	if err != nil {
 		log.Fatalln("download failed", err)
 	}
-	_, err = http.Instance.DownloadFile("https://cdn.modrinth.com/data/KmiWHzQ4/versions/1.5.0/Skyblocker-Modpack.mrpack", os.TempDir(), "")
+	_, err = requester.DefaultHttpClient.DownloadFile("https://cdn.modrinth.com/data/KmiWHzQ4/versions/1.5.0/Skyblocker-Modpack.mrpack", os.TempDir(), "")
 	if err != nil {
 		log.Fatalln("download failed", err)
 	}

mojang/api.go

@@ -1,7 +1,7 @@
 package mojang
 
 import (
-	"github.com/nothub/mrpack-install/http"
+	"github.com/nothub/mrpack-install/requester"
 )
 
 const manifestUrl = "https://launchermeta.mojang.com/mc/game/version_manifest.json"
@@ -13,7 +13,7 @@ func LatestVersion() (string, error) {
 			Snapshot string `json:"snapshot"`
 		} `json:"latest"`
 	}
-	err := http.Instance.GetJson(manifestUrl, &manifest, nil)
+	err := requester.DefaultHttpClient.GetJson(manifestUrl, &manifest, nil)
 	if err != nil {
 		return "", err
 	}

http/client.go → requester/client.go

@@ -1,4 +1,4 @@
-package http
+package requester
 
 import (
 	"encoding/json"
@@ -17,20 +17,20 @@ type ErrorModel interface {
 	String() string
 }
 
-var Instance = NewHTTPClient()
+var DefaultHttpClient = NewHTTPClient()
 
-func (client *Client) GetJson(url string, respModel interface{}, errModel ErrorModel) error {
+func (httpClient *HTTPClient) GetJson(url string, respModel interface{}, errModel ErrorModel) error {
 	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return err
 	}
 
-	request.Header.Set("User-Agent", client.UserAgent)
+	request.Header.Set("User-Agent", httpClient.UserAgent)
 	request.Header.Set("Accept", "application/json")
 
 	request.Close = true
 
-	response, err := client.Do(request)
+	response, err := httpClient.Do(request)
 	if err != nil {
 		return err
 	}
@@ -57,17 +57,17 @@ func (client *Client) GetJson(url string, respModel interface{}, errModel ErrorM
 	return nil
 }
 
-func (client *Client) DownloadFile(url string, downloadDir string, fileName string) (string, error) {
+func (httpClient *HTTPClient) DownloadFile(url string, downloadDir string, fileName string) (string, error) {
 	// TODO: hashsum based local file cache
 
 	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return "", err
 	}
-	request.Header.Set("User-Agent", client.UserAgent)
+	request.Header.Set("User-Agent", httpClient.UserAgent)
 	request.Close = true
 
-	response, err := client.Do(request)
+	response, err := httpClient.Do(request)
 	if err != nil {
 		return "", err
 	}

requester/client_test.go

@@ -0,0 +1,3 @@
+package requester
+
+// TODO