diff --git a/cmd/root.go b/cmd/root.go index 09d159d..3c2bddc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,9 +3,10 @@ package cmd import ( "fmt" "github.com/nothub/mrpack-install/buildinfo" + "github.com/nothub/mrpack-install/http" + "github.com/nothub/mrpack-install/http/download" modrinth "github.com/nothub/mrpack-install/modrinth/api" "github.com/nothub/mrpack-install/modrinth/mrpack" - "github.com/nothub/mrpack-install/requester" "github.com/nothub/mrpack-install/server" "github.com/nothub/mrpack-install/update" "github.com/nothub/mrpack-install/util" @@ -34,12 +35,12 @@ func init() { } type GlobalOpts struct { - Host string - ServerDir string - ServerFile string - Proxy string - DownloadThreads int - RetryTimes int + Host string + ServerDir string + ServerFile string + Proxy string + DlThreads int + DlRetries int } func GlobalOptions(cmd *cobra.Command) *GlobalOpts { @@ -78,27 +79,26 @@ func GlobalOptions(cmd *cobra.Command) *GlobalOpts { log.Fatalln(err) } if proxy != "" { - // TODO: stop changing the default http client - err := requester.DefaultHttpClient.SetProxy(proxy) + err := http.DefaultClient.SetProxy(proxy) if err != nil { log.Fatalln(err) } } opts.Proxy = proxy - downloadThreads, err := cmd.Flags().GetInt("download-threads") - if err != nil || downloadThreads > 64 { - downloadThreads = 8 + dlThreads, err := cmd.Flags().GetInt("download-threads") + if err != nil || dlThreads > 64 { + dlThreads = 8 fmt.Println(err) } - opts.DownloadThreads = downloadThreads + opts.DlThreads = dlThreads retryTimes, err := cmd.Flags().GetInt("download-retries") if err != nil { retryTimes = 3 fmt.Println(err) } - opts.RetryTimes = retryTimes + opts.DlRetries = retryTimes return &opts } @@ -147,16 +147,15 @@ var rootCmd = &cobra.Command{ } index, zipPath := handleArgs(input, version, opts.ServerDir, opts.Host) + fmt.Printf("Installing %q from %q to %q", index.Name, zipPath, opts.ServerDir) + for _, file := range index.Files { util.AssertPathSafe(file.Path, opts.ServerDir) } - fmt.Println("Installing:", index.Name) - // download server if not present if !util.PathIsFile(path.Join(opts.ServerDir, opts.ServerFile)) { fmt.Println("Server file not present, downloading...\n(Point --server-dir and --server-file flags to an existing server file to skip this step.)") - inst := server.InstallerFromDeps(&index.Deps) err := inst.Install(opts.ServerDir, opts.ServerFile) if err != nil { @@ -166,26 +165,15 @@ var rootCmd = &cobra.Command{ fmt.Println("Server file already present, proceeding...") } - // mod downloads - fmt.Printf("Downloading %v dependencies...\n", len(index.Files)) - var downloads []*requester.Download - for i := range index.Files { - file := index.Files[i] - if file.Env.Server == modrinth.UnsupportedEnvSupport { - continue - } - downloads = append(downloads, requester.NewDownload(file.Downloads, map[string]string{"sha1": file.Hashes.Sha1}, filepath.Base(file.Path), path.Join(opts.ServerDir, filepath.Dir(file.Path)))) - } - downloadPools := requester.NewDownloadPools(requester.DefaultHttpClient, downloads, opts.DownloadThreads, opts.RetryTimes) - downloadPools.Do() - modsUnclean := false - for i := range downloadPools.Downloads { - dl := downloadPools.Downloads[i] - if !dl.Success { - modsUnclean = true - fmt.Println("Dependency downloaded Fail:", dl.FileName) - } + // downloads + downloads := index.ServerDownloads() + fmt.Printf("Downloading %v dependencies...\n", len(downloads)) + downloader := download.Downloader{ + Downloads: downloads, + Threads: opts.DlThreads, + Retries: opts.DlRetries, } + downloader.Download(opts.ServerDir) // overrides fmt.Println("Extracting overrides...") @@ -194,6 +182,7 @@ var rootCmd = &cobra.Command{ log.Fatalln(err) } + // save state file packState, err := update.BuildPackState(index, zipPath) if err != nil { log.Fatalln(err) @@ -203,29 +192,12 @@ var rootCmd = &cobra.Command{ log.Fatalln(err) } - if modsUnclean { - fmt.Println("There have been problems downloading mods, you probably have to fix some dependency problems manually!") - } + util.RemoveEmptyDirs(opts.ServerDir) - fmt.Println("Done :) Have a nice day ✌️") + fmt.Println("Installation done :) Have a nice day ✌️") }, } -func readArgs(args []string) (string, string) { - var input string - var version string - - if len(args) > 0 { - input = args[0] - } - - if len(args) > 1 { - version = args[1] - } - - return input, version -} - func handleArgs(input string, version string, serverDir string, host string) (*mrpack.Index, string) { err := os.MkdirAll(serverDir, 0755) if err != nil { @@ -238,7 +210,7 @@ func handleArgs(input string, version string, serverDir string, host string) (*m } else if util.IsValidUrl(input) { fmt.Println("Downloading mrpack file from", input) - file, err := requester.DefaultHttpClient.DownloadFile(input, serverDir, "") + file, err := http.DefaultClient.DownloadFile(input, serverDir, "") if err != nil { log.Fatalln(err.Error()) } @@ -280,7 +252,7 @@ func handleArgs(input string, version string, serverDir string, host string) (*m for i := range files { if strings.HasSuffix(files[i].Filename, ".mrpack") { fmt.Println("Downloading mrpack file from", files[i].Url) - file, err := requester.DefaultHttpClient.DownloadFile(files[i].Url, serverDir, "") + file, err := http.DefaultClient.DownloadFile(files[i].Url, serverDir, "") if err != nil { // TODO: check next file on failure log.Fatalln(err.Error()) diff --git a/cmd/update.go b/cmd/update.go index d537974..8674eab 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -53,6 +53,6 @@ var updateCmd = &cobra.Command{ index, zipPath := handleArgs(input, version, opts.ServerDir, opts.Host) - update.Cmd(opts, index, zipPath) + update.Cmd(opts.ServerDir, opts.DlThreads, opts.DlRetries, index, zipPath) }, } diff --git a/go.mod b/go.mod index 27ac04d..0377f20 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/google/uuid v1.3.0 github.com/nothub/hashutils v0.4.0 github.com/spf13/cobra v1.6.1 + golang.org/x/exp v0.0.0-20230212135524-a684f29349b6 ) require ( diff --git a/go.sum b/go.sum index 622c570..ad7681c 100644 --- a/go.sum +++ b/go.sum @@ -10,5 +10,7 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/exp v0.0.0-20230212135524-a684f29349b6 h1:Ic9KukPQ7PegFzHckNiMTQXGgEszA7mY2Fn4ZMtnMbw= +golang.org/x/exp v0.0.0-20230212135524-a684f29349b6/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/http/client.go b/http/client.go new file mode 100644 index 0000000..1de113d --- /dev/null +++ b/http/client.go @@ -0,0 +1,73 @@ +package http + +import ( + "fmt" + "github.com/nothub/mrpack-install/buildinfo" + "net" + "net/http" + "net/url" + "runtime/debug" + "time" +) + +type Client struct { + c http.Client + ua string +} + +var DefaultClient = newHTTPClient() + +func newHTTPClient() *Client { + c := &Client{ + c: http.Client{}, + } + + c.c.Transport = newTransport() + + c.ua = fmt.Sprintf("%s/%s", "mrpack-install", buildinfo.Version) + info, ok := debug.ReadBuildInfo() + if ok && info.Main.Path != "" { + c.ua = fmt.Sprintf("%s (+https://%s)", c.ua, info.Main.Path) + } + + return c +} + +func newTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }.DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 20 * time.Second, + ResponseHeaderTimeout: 25 * time.Second, + ExpectContinueTimeout: 10 * time.Second, + } +} + +func (c *Client) SetProxy(CustomProxy string) error { + proxy, err := url.Parse(CustomProxy) + if err != nil { + return err + } + + transport := newTransport() + transport.Proxy = http.ProxyURL(proxy) + c.c.Transport = transport + + // Test proxy + httpUrl := "https://api.modrinth.com/" + response, err := c.c.Get(httpUrl) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return err + } + + return nil +} diff --git a/http/download/multi.go b/http/download/multi.go new file mode 100644 index 0000000..cbeb7a7 --- /dev/null +++ b/http/download/multi.go @@ -0,0 +1,66 @@ +package download + +import ( + "crypto" + "github.com/nothub/hashutils/chksum" + "github.com/nothub/hashutils/encoding" + "github.com/nothub/mrpack-install/http" + modrinth "github.com/nothub/mrpack-install/modrinth/api" + "log" + "path" + "path/filepath" + "sync" +) + +type Download struct { + Path string + Urls []string + Hashes modrinth.Hashes +} + +type Downloader struct { + Downloads []*Download + Threads int + Retries int +} + +func (g *Downloader) Download(baseDir string) { + var wg sync.WaitGroup + for i := range g.Downloads { + wg.Add(1) + dl := g.Downloads[i] + go func() { + defer wg.Done() + absPath, _ := filepath.Abs(path.Join(baseDir, dl.Path)) + success := false + for _, link := range dl.Urls { + // retry when download failed + for retries := 0; retries < g.Retries; retries++ { + // try download + f, err := http.DefaultClient.DownloadFile(link, path.Dir(absPath), path.Base(absPath)) + if err != nil { + log.Printf("Download failed for %s (attempt %v), because: %s\n", dl.Path, retries+1, err.Error()) + continue + } + // check hashcode + _, err = chksum.VerifyFile(f, dl.Hashes.Sha512, crypto.SHA512.New(), encoding.Hex) + if err != nil { + log.Printf("Hash check failed for %s (attempt %v), because: %s\n", dl.Path, retries+1, err.Error()) + continue + } + // success yay + log.Printf("Downloaded: %s\n", f) + success = true + break + } + if success { + break + } + } + if !success { + log.Printf("Downloaded failed: %s\n", dl.Path) + } + }() + } + wg.Wait() +} diff --git a/requester/client.go b/http/features.go similarity index 78% rename from requester/client.go rename to http/features.go index 89cd947..f715778 100644 --- a/requester/client.go +++ b/http/features.go @@ -1,4 +1,4 @@ -package requester +package http import ( "encoding/json" @@ -13,18 +13,16 @@ import ( "strconv" ) -var DefaultHttpClient = NewHTTPClient() - -func (httpClient *HTTPClient) GetJson(url string, respModel interface{}, errModel error) error { +func (c *Client) GetJson(url string, respModel interface{}, errModel error) error { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return err } - req.Header.Set("User-Agent", httpClient.UserAgent) + req.Header.Set("User-Agent", c.ua) req.Header.Set("Accept", "application/json") req.Close = true - res, err := httpClient.sendRequest(req) + res, err := c.sendRequest(req) if err != nil { return err } @@ -51,17 +49,19 @@ func (httpClient *HTTPClient) GetJson(url string, respModel interface{}, errMode return nil } -func (httpClient *HTTPClient) DownloadFile(url string, downloadDir string, fileName string) (string, error) { +func (c *Client) DownloadFile(url string, downloadDir string, fileName string) (string, error) { // TODO: hashsum based local file cache + // TODO: this needs to (silently?) overwrite existing files! + request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return "", err } - request.Header.Set("User-Agent", httpClient.UserAgent) + request.Header.Set("User-Agent", c.ua) request.Close = true - response, err := httpClient.sendRequest(request) + response, err := c.sendRequest(request) if err != nil { return "", err } @@ -110,9 +110,9 @@ func (httpClient *HTTPClient) DownloadFile(url string, downloadDir string, fileN return file.Name(), nil } -func (httpClient *HTTPClient) sendRequest(request *http.Request) (*http.Response, error) { +func (c *Client) sendRequest(request *http.Request) (*http.Response, error) { awaitRateLimits(request.Host) - response, err := httpClient.Do(request) + response, err := c.c.Do(request) if err != nil { return nil, err } diff --git a/requester/ratelimits.go b/http/ratelimits.go similarity index 98% rename from requester/ratelimits.go rename to http/ratelimits.go index 1f21107..a5b5daa 100644 --- a/requester/ratelimits.go +++ b/http/ratelimits.go @@ -1,4 +1,4 @@ -package requester +package http import ( "fmt" diff --git a/modrinth/api/client.go b/modrinth/api/client.go index 32d4d2a..4ae781f 100644 --- a/modrinth/api/client.go +++ b/modrinth/api/client.go @@ -1,19 +1,19 @@ package api import ( - "github.com/nothub/mrpack-install/requester" + "github.com/nothub/mrpack-install/http" "log" "net/url" ) type ModrinthClient struct { - Http *requester.HTTPClient + Http *http.Client BaseUrl string } func NewClient(host string) *ModrinthClient { client := ModrinthClient{ - Http: requester.DefaultHttpClient, + Http: http.DefaultClient, } u, err := url.Parse("https://" + host + "/") if err != nil { diff --git a/modrinth/mrpack/index.go b/modrinth/mrpack/index.go index ac88051..fe93ce5 100644 --- a/modrinth/mrpack/index.go +++ b/modrinth/mrpack/index.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/nothub/mrpack-install/http/download" "io" ) @@ -100,21 +101,23 @@ func ReadIndex(zipFile string) (*Index, error) { return &index, nil } -type Downloads map[string]string - -func (index *Index) ServerDownloads() (*Downloads, error) { - downloads := make(Downloads, len(index.Files)) - +func (index *Index) ServerDownloads() []*download.Download { + var downloads []*download.Download for _, file := range index.Files { if file.Env.Server == modrinth.UnsupportedEnvSupport { continue } + if len(file.Downloads) < 1 { fmt.Printf("No downloads for file: %s\n", file.Path) continue } - downloads[file.Path] = file.Downloads[0] - } - return &downloads, nil + downloads = append(downloads, &download.Download{ + Path: file.Path, + Urls: file.Downloads, + Hashes: file.Hashes, + }) + } + return downloads } diff --git a/mojang/api.go b/mojang/api.go index e0dfb0b..93bb3eb 100644 --- a/mojang/api.go +++ b/mojang/api.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "errors" "github.com/google/uuid" - "github.com/nothub/mrpack-install/requester" + "github.com/nothub/mrpack-install/http" "time" ) @@ -112,7 +112,7 @@ type Player struct { func GetManifest() (*Manifest, error) { var manifest Manifest - err := requester.DefaultHttpClient.GetJson(manifestUrl, &manifest, nil) + err := http.DefaultClient.GetJson(manifestUrl, &manifest, nil) if err != nil { return nil, err } @@ -129,7 +129,7 @@ func LatestRelease() (string, error) { func GetMeta(version string) (*Meta, error) { var manifest Manifest - err := requester.DefaultHttpClient.GetJson(manifestUrl, &manifest, nil) + err := http.DefaultClient.GetJson(manifestUrl, &manifest, nil) if err != nil { return nil, err } @@ -148,7 +148,7 @@ func GetMeta(version string) (*Meta, error) { } var meta Meta - err = requester.DefaultHttpClient.GetJson(u, &meta, nil) + err = http.DefaultClient.GetJson(u, &meta, nil) if err != nil { return nil, err } @@ -158,7 +158,7 @@ func GetMeta(version string) (*Meta, error) { func GetPlayer(name string) (*Player, error) { var player Player - err := requester.DefaultHttpClient.GetJson(playerUrl(name), &player, nil) + err := http.DefaultClient.GetJson(playerUrl(name), &player, nil) if err != nil { return nil, err } diff --git a/requester/http_client.go b/requester/http_client.go deleted file mode 100644 index d8ddb35..0000000 --- a/requester/http_client.go +++ /dev/null @@ -1,117 +0,0 @@ -package requester - -import ( - "crypto/tls" - "fmt" - "github.com/nothub/mrpack-install/buildinfo" - "net/http" - "net/http/cookiejar" - "net/url" - "runtime/debug" - "time" -) - -type HTTPClient struct { - http.Client - UserAgent string - transport *http.Transport -} - -func NewHTTPClient() *HTTPClient { - httpClient := &HTTPClient{ - Client: http.Client{}, - } - - httpClient.Client.Jar, _ = cookiejar.New(nil) - - httpClient.UserAgent = fmt.Sprintf("%s/%s", "mrpack-install", buildinfo.Version) - info, ok := debug.ReadBuildInfo() - if ok && info.Main.Path != "" { - httpClient.UserAgent = fmt.Sprintf("%s (+https://%s)", httpClient.UserAgent, info.Main.Path) - } - - return httpClient -} - -func (httpClient *HTTPClient) lazyInit() { - if httpClient.transport == nil { - httpClient.transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: false, - }, - TLSHandshakeTimeout: 20 * time.Second, - DisableKeepAlives: false, - DisableCompression: false, // gzip - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - ResponseHeaderTimeout: 25 * time.Second, - ExpectContinueTimeout: 10 * time.Second, - } - httpClient.Client.Transport = httpClient.transport - } -} - -func (httpClient *HTTPClient) SetUserAgent(ua string) { - httpClient.UserAgent = ua -} - -func (httpClient *HTTPClient) SetCookiejar(jar http.CookieJar) { - httpClient.Client.Jar = jar -} - -func (httpClient *HTTPClient) ResetCookiejar() { - httpClient.Jar, _ = cookiejar.New(nil) -} - -func (httpClient *HTTPClient) SetProxy(CustomProxy string) error { - httpClient.lazyInit() - proxy, err := url.Parse(CustomProxy) - if err != nil { - return err - } - - httpClient.transport.Proxy = http.ProxyURL(proxy) - - // Test proxy - httpUrl := "https://api.modrinth.com/" - response, err := httpClient.Get(httpUrl) - if err != nil { - return err - } - if response.StatusCode != http.StatusOK { - return err - } - return nil -} - -func (httpClient *HTTPClient) SetInsecureSkipVerify(b bool) { - httpClient.lazyInit() - httpClient.transport.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: b, - } -} - -func (httpClient *HTTPClient) SetKeepAlive(b bool) { - httpClient.lazyInit() - httpClient.transport.DisableKeepAlives = !b -} - -func (httpClient *HTTPClient) SetGzip(b bool) { - httpClient.lazyInit() - httpClient.transport.DisableCompression = !b -} - -func (httpClient *HTTPClient) SetResponseHeaderTimeout(t time.Duration) { - httpClient.lazyInit() - httpClient.transport.ResponseHeaderTimeout = t -} - -func (httpClient *HTTPClient) SetTLSHandshakeTimeout(t time.Duration) { - httpClient.lazyInit() - httpClient.transport.TLSHandshakeTimeout = t -} - -func (httpClient *HTTPClient) SetTimeout(t time.Duration) { - httpClient.Client.Timeout = t -} diff --git a/requester/multi_download.go b/requester/multi_download.go deleted file mode 100644 index 30538a6..0000000 --- a/requester/multi_download.go +++ /dev/null @@ -1,77 +0,0 @@ -package requester - -import ( - "crypto" - "fmt" - "github.com/nothub/hashutils/chksum" - "github.com/nothub/hashutils/encoding" - "sync" -) - -type Download struct { - links []string - hashes map[string]string - FileName string - downloadDir string - Success bool -} - -type DownloadPools struct { - httpClient *HTTPClient - Downloads []*Download - threads int - maxRetries int -} - -func NewDownloadPools(httpClient *HTTPClient, downloads []*Download, threads int, maxRetries int) *DownloadPools { - return &DownloadPools{httpClient, downloads, threads, maxRetries} -} - -func NewDownload(links []string, hashes map[string]string, fileName string, downloadDir string) *Download { - return &Download{links, hashes, fileName, downloadDir, false} -} - -func (downloadPools *DownloadPools) Do() { - var wg sync.WaitGroup - ch := make(chan struct{}, downloadPools.threads) - for i := range downloadPools.Downloads { - dl := downloadPools.Downloads[i] - - //goroutine - ch <- struct{}{} - wg.Add(1) - go func() { - defer wg.Done() - for _, link := range dl.links { - // retry when download failed - for retries := 0; retries < downloadPools.maxRetries; retries++ { - - // download file - f, err := downloadPools.httpClient.DownloadFile(link, dl.downloadDir, dl.FileName) - if err != nil { - fmt.Println("Download failed for:", dl.FileName, err, "attempt:", retries+1) - continue - } - - // check hashcode - if sha1code, ok := dl.hashes["sha1"]; ok { - _, err = chksum.VerifyFile(f, sha1code, crypto.SHA1.New(), encoding.Hex) - } - if err != nil { - fmt.Println("Hash check failed for:", dl.FileName, err, "attempt:", retries+1) - continue - } - - fmt.Println("Downloaded:", f) - dl.Success = true - break - } - if dl.Success { - break - } - } - <-ch - }() - } - wg.Wait() -} diff --git a/server/fabric.go b/server/fabric.go index b801623..bcb8cee 100644 --- a/server/fabric.go +++ b/server/fabric.go @@ -3,7 +3,7 @@ package server import ( "errors" "fmt" - "github.com/nothub/mrpack-install/requester" + "github.com/nothub/mrpack-install/http" "net/url" ) @@ -33,7 +33,7 @@ func (inst *FabricInstaller) Install(serverDir string, serverFile string) error return err } - file, err := requester.DefaultHttpClient.DownloadFile(u.String(), serverDir, serverFile) + file, err := http.DefaultClient.DownloadFile(u.String(), serverDir, serverFile) if err != nil { return err } @@ -49,7 +49,7 @@ func latestFabricLoaderVersion(mcVer string) (string, error) { Stable bool `json:"stable"` } `json:"loader"` } - err := requester.DefaultHttpClient.GetJson("https://meta.fabricmc.net/v2/versions/loader/"+mcVer, &loaders, nil) + err := http.DefaultClient.GetJson("https://meta.fabricmc.net/v2/versions/loader/"+mcVer, &loaders, nil) if err != nil { return "", err } @@ -66,7 +66,7 @@ func latestFabricInstallerVersion() (string, error) { Version string `json:"version"` Stable bool `json:"stable"` } - err := requester.DefaultHttpClient.GetJson("https://meta.fabricmc.net/v2/versions/installer", &installers, nil) + err := http.DefaultClient.GetJson("https://meta.fabricmc.net/v2/versions/installer", &installers, nil) if err != nil { return "", err } diff --git a/server/paper.go b/server/paper.go index 21e3b7d..b217329 100644 --- a/server/paper.go +++ b/server/paper.go @@ -3,7 +3,7 @@ package server import ( "errors" "fmt" - "github.com/nothub/mrpack-install/requester" + "github.com/nothub/mrpack-install/http" "strconv" ) @@ -25,7 +25,7 @@ func (inst *PaperInstaller) Install(serverDir string, serverFile string) error { } `json:"builds"` } - err := requester.DefaultHttpClient.GetJson("https://api.papermc.io/v2/projects/paper/versions/"+ + err := http.DefaultClient.GetJson("https://api.papermc.io/v2/projects/paper/versions/"+ inst.MinecraftVersion+"/builds", &response, nil) if err != nil { return err @@ -37,7 +37,7 @@ func (inst *PaperInstaller) Install(serverDir string, serverFile string) error { u := "https://api.papermc.io/v2/projects/paper/versions/" + inst.MinecraftVersion + "/builds/" + strconv.Itoa(response.Builds[i].Id) + "/downloads/" + response.Builds[i].Downloads.Application.Name - file, err := requester.DefaultHttpClient.DownloadFile(u, serverDir, serverFile) + file, err := http.DefaultClient.DownloadFile(u, serverDir, serverFile) if err != nil { return err } diff --git a/server/quilt.go b/server/quilt.go index 1d74322..cf90f91 100644 --- a/server/quilt.go +++ b/server/quilt.go @@ -3,7 +3,7 @@ package server import ( "errors" "fmt" - "github.com/nothub/mrpack-install/requester" + "github.com/nothub/mrpack-install/http" "github.com/nothub/mrpack-install/util" "os" "os/exec" @@ -18,7 +18,7 @@ type QuiltInstaller struct { } func (inst *QuiltInstaller) Install(serverDir string, serverFile string) error { - installer, err := requester.DefaultHttpClient.DownloadFile(quiltInstallerUrl, ".", "") + installer, err := http.DefaultClient.DownloadFile(quiltInstallerUrl, ".", "") if err != nil { return err } diff --git a/server/vanilla.go b/server/vanilla.go index 6547747..bad74a0 100644 --- a/server/vanilla.go +++ b/server/vanilla.go @@ -5,8 +5,8 @@ import ( "errors" "github.com/nothub/hashutils/chksum" "github.com/nothub/hashutils/encoding" + "github.com/nothub/mrpack-install/http" "github.com/nothub/mrpack-install/mojang" - "github.com/nothub/mrpack-install/requester" "log" ) @@ -20,7 +20,7 @@ func (inst *VanillaInstaller) Install(serverDir string, serverFile string) error return err } - file, err := requester.DefaultHttpClient.DownloadFile(meta.Downloads.Server.Url, serverDir, serverFile) + file, err := http.DefaultClient.DownloadFile(meta.Downloads.Server.Url, serverDir, serverFile) if err != nil { log.Fatalln(err) } diff --git a/update/actions.go b/update/actions.go deleted file mode 100644 index dd3a18a..0000000 --- a/update/actions.go +++ /dev/null @@ -1,123 +0,0 @@ -package update - -import ( - "archive/zip" - "crypto" - "fmt" - "github.com/nothub/hashutils/chksum" - "github.com/nothub/hashutils/encoding" - "github.com/nothub/mrpack-install/requester" - "github.com/nothub/mrpack-install/util" - "io" - "os" - "path/filepath" - "strings" -) - -type strategy uint8 - -const ( - Delete strategy = iota - Backup - NoOp -) - -// GetStrategy selects one of 3 strategies for handling old files: -// -// 1. NoOp - File does not exist -// -// 2. Delete - File exists and hash values match -// -// 3. Backup - File exists but hash values do not match -// -// Hash must be sha512 and hex encoded. -func GetStrategy(hash string, path string) strategy { - if !util.PathIsFile(path) { - return NoOp - } - match, _ := chksum.VerifyFile(path, hash, crypto.SHA512.New(), encoding.Hex) - if match { - return Delete - } else { - return Backup - } -} - -// ShouldBackup indicates if the file exists but the hash value does not match. -func ShouldBackup(path string, hash string) bool { - return GetStrategy(hash, path) == Backup -} - -func Do(newFiles []File, serverDir string, zipPath string, threads int, retries int) error { - - var downloads []*requester.Download - downloadPools := requester.NewDownloadPools(requester.DefaultHttpClient, downloads, threads, retries) - - for filePath := range newFiles { - downloadPools.Downloads = append(downloadPools.Downloads, requester.NewDownload(hashes[filePath].DownloadLink, map[string]string{"sha1": hashes[filePath]}, filepath.Base(filePath), filepath.Join(serverDir, filepath.Dir(filePath)))) - } - downloadPools.Do() - - // unzip update file - r, err := zip.OpenReader(zipPath) - if err != nil { - return err - } - defer func(r *zip.ReadCloser) { - err := r.Close() - if err != nil { - fmt.Println(err) - } - }(r) - - for _, f := range r.File { - if f.FileInfo().IsDir() { - continue - } - - filePathInZip := f.Name - if strings.HasPrefix(filePathInZip, "overrides/") { - filePathInZip = strings.TrimPrefix(filePathInZip, "overrides/") - } else if strings.HasPrefix(filePathInZip, "server-overrides/") { - filePathInZip = strings.TrimPrefix(filePathInZip, "server-overrides/") - } else { - continue - } - - if _, ok := hashes[filePathInZip]; ok && hashes[filePathInZip].DownloadLink == nil { - - targetPath := filepath.Join(serverDir, filePathInZip) - - err := os.MkdirAll(filepath.Dir(targetPath), 0755) - if err != nil { - return err - } - - fileReader, err := f.Open() - if err != nil { - return err - } - - outFile, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - if _, err := io.Copy(outFile, fileReader); err != nil { - return err - } - - err = fileReader.Close() - if err != nil { - return err - } - err = outFile.Close() - if err != nil { - return err - } - - fmt.Println("Override file extracted:", targetPath) - - } - } - return nil -} diff --git a/update/command.go b/update/command.go index 11030a4..abac346 100644 --- a/update/command.go +++ b/update/command.go @@ -2,19 +2,20 @@ package update import ( "fmt" - "github.com/nothub/mrpack-install/cmd" + "github.com/nothub/mrpack-install/http/download" "github.com/nothub/mrpack-install/modrinth/mrpack" "github.com/nothub/mrpack-install/update/backup" "github.com/nothub/mrpack-install/util" "log" - "path/filepath" "reflect" ) -func Cmd(opts *cmd.UpdateOpts, index *mrpack.Index, zipPath string) { - fmt.Printf("Updating %s with %s", opts.ServerDir, zipPath) +import "golang.org/x/exp/slices" - oldState, err := LoadPackState(opts.ServerDir) +func Cmd(serverDir string, dlThreads int, dlRetries int, index *mrpack.Index, zipPath string) { + fmt.Printf("Updating %q in %q with %q", index.Name, serverDir, zipPath) + + oldState, err := LoadPackState(serverDir) if err != nil { log.Fatalln(err) } @@ -23,8 +24,8 @@ func Cmd(opts *cmd.UpdateOpts, index *mrpack.Index, zipPath string) { if err != nil { log.Fatalln(err) } - for filePath := range newState.Files { - util.AssertPathSafe(filePath, opts.ServerDir) + for filePath := range newState.Hashes { + util.AssertPathSafe(filePath, serverDir) } if !reflect.DeepEqual(oldState.Deps, newState.Deps) { @@ -32,55 +33,66 @@ func Cmd(opts *cmd.UpdateOpts, index *mrpack.Index, zipPath string) { log.Fatalln("mismatched versions, please upgrade manually") } - for path := range oldState.Files { - // ignore unchanged files - if newState.Files[path] == oldState.Files[path] { - delete(oldState.Files, path) - delete(newState.Files, path) - } - // skip deletion of old files that we overwrite with new files - if _, found := newState.Files[path]; found { - delete(oldState.Files, path) + // ignore files that are left unchanged in the update process + var ignores []string + for path := range newState.Hashes { + if newState.Hashes[path] == oldState.Hashes[path] { + ignores = append(ignores, path) } } - for path, hashes := range oldState.Files { - if ShouldBackup(path, hashes.Sha512) { - err := backup.Create(path, opts.ServerDir) - if err != nil { - log.Fatalln(err.Error()) - } + // backup if the file exists but the new hash value does not match + for path := range oldState.Hashes { + if slices.Contains(ignores, path) { + continue + } + + if !util.PathIsFile(path) { + continue + } + + // check if file will be replaced + _, ok := newState.Hashes[path] + if !ok { + continue + } + + err := backup.Create(path, serverDir) + if err != nil { + log.Fatalln(err.Error()) } } - // TODO: correctly handle new files - var newFiles []mrpack.File - for path, hashes := range newState.Files { - var f mrpack.File - f.Path = path - switch GetStrategy(hashes.Sha512, filepath.Join(opts.ServerDir, path)) { - case Delete: - delete(newState.Files, path) - case Backup: - err := backup.Create(path, opts.ServerDir) - if err != nil { - log.Fatalln(err.Error()) - } + // downloads + var downloads []*download.Download + for _, dl := range index.ServerDownloads() { + if !slices.Contains(ignores, dl.Path) { + downloads = append(downloads, dl) } - newFiles = append(newFiles, f) } - err = Do(newFiles, opts.ServerDir, zipPath, opts.DownloadThreads, opts.RetryTimes) + fmt.Printf("Downloading %v dependencies...\n", len(downloads)) + downloader := download.Downloader{ + Downloads: downloads, + Threads: dlThreads, + Retries: dlRetries, + } + downloader.Download(serverDir) + + // overrides + fmt.Println("Extracting overrides...") + err = mrpack.ExtractOverrides(zipPath, serverDir) if err != nil { log.Fatalln(err) } - util.RemoveEmptyDirs(opts.ServerDir) - - err = newState.Save(opts.ServerDir) + // save state file + err = newState.Save(serverDir) if err != nil { log.Fatalln(err) } - fmt.Println("Done :) Have a nice day ✌️") + util.RemoveEmptyDirs(serverDir) + + fmt.Println("Update finished :) Have a nice day ✌️") } diff --git a/update/packstate.go b/update/packstate.go index ef8048a..6ba733e 100644 --- a/update/packstate.go +++ b/update/packstate.go @@ -17,7 +17,7 @@ type PackState struct { Name string `json:"name"` Version string `json:"version"` Deps mrpack.Deps `json:"dependencies"` - Files map[string]modrinth.Hashes `json:"files"` + Hashes map[string]modrinth.Hashes `json:"hashes"` } func (state *PackState) Save(serverDir string) error { @@ -60,19 +60,19 @@ func BuildPackState(index *mrpack.Index, zipPath string) (*PackState, error) { state.Name = index.Name state.Version = index.Version state.Deps = index.Deps - state.Files = make(map[string]modrinth.Hashes) + state.Hashes = make(map[string]modrinth.Hashes) - // downloads + // download hashes for _, indexFile := range index.Files { if indexFile.Env.Server == modrinth.UnsupportedEnvSupport { continue } - state.Files[indexFile.Path] = indexFile.Hashes + state.Hashes[indexFile.Path] = indexFile.Hashes } - // overrides + // override hashes for p, hashes := range mrpack.OverrideHashes(zipPath) { - state.Files[p] = hashes + state.Hashes[p] = hashes } return &state, nil