Skip to content

Commit

Permalink
Merge pull request #108 from WillAbides/allow-checksums
Browse files Browse the repository at this point in the history 
Add --allow-missing-coverage
  • Loading branch information
@WillAbides
WillAbides authored Mar 11, 2023
2 parents 3a5a4e7 + 510eea0 commit 0a66f04
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 48 deletions.
41 changes: 24 additions & 17 deletions cmd/bindown/cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ var kongVars = kong.Vars{
"download_force_help": `force download even if the file already exists`,
"download_target_file_help": `filename and path for the downloaded file. Default downloads to cache.`,
"download_dependency_help": `name of the dependency to download`,
"allow_missing_checksum": `allow missing checksums`,
"download_help": `download a dependency but don't extract or install it`,
"extract_dependency_help": `name of the dependency to extract`,
"extract_help": `download and extract a dependency but don't install it`,
Expand Down Expand Up @@ -180,10 +181,11 @@ func (d validateCmd) Run(ctx context.Context) error {
}

type installCmd struct {
Force bool `kong:"help=${install_force_help}"`
Dependency string `kong:"required=true,arg,help=${download_dependency_help},predictor=bin"`
TargetFile string `kong:"type=path,name=output,type=file,help=${install_target_file_help}"`
System bindown.SystemInfo `kong:"name=system,default=${system_default},help=${system_help},predictor=allSystems"`
Force bool `kong:"help=${install_force_help}"`
Dependency string `kong:"required=true,arg,help=${download_dependency_help},predictor=bin"`
TargetFile string `kong:"type=path,name=output,type=file,help=${install_target_file_help}"`
System bindown.SystemInfo `kong:"name=system,default=${system_default},help=${system_help},predictor=allSystems"`
AllowMissingChecksum bool `kong:"name=allow-missing-checksum,help=${allow_missing_checksum}"`
}

func (d *installCmd) Run(kctx *kong.Context) error {
Expand All @@ -193,8 +195,9 @@ func (d *installCmd) Run(kctx *kong.Context) error {
return err
}
pth, err := config.InstallDependency(d.Dependency, d.System, &bindown.ConfigInstallDependencyOpts{
TargetPath: d.TargetFile,
Force: d.Force,
TargetPath: d.TargetFile,
Force: d.Force,
AllowMissingChecksum: d.AllowMissingChecksum,
})
if err != nil {
return err
Expand All @@ -204,10 +207,11 @@ func (d *installCmd) Run(kctx *kong.Context) error {
}

type downloadCmd struct {
Force bool `kong:"help=${download_force_help}"`
System bindown.SystemInfo `kong:"name=system,default=${system_default},help=${system_help},predictor=allSystems"`
Dependency string `kong:"required=true,arg,help=${download_dependency_help},predictor=bin"`
TargetFile string `kong:"name=output,help=${download_target_file_help}"`
Force bool `kong:"help=${download_force_help}"`
System bindown.SystemInfo `kong:"name=system,default=${system_default},help=${system_help},predictor=allSystems"`
Dependency string `kong:"required=true,arg,help=${download_dependency_help},predictor=bin"`
TargetFile string `kong:"name=output,help=${download_target_file_help}"`
AllowMissingChecksum bool `kong:"name=allow-missing-checksum,help=${allow_missing_checksum}"`
}

func (d *downloadCmd) Run(ctx context.Context, kctx *kong.Context) error {
Expand All @@ -216,8 +220,9 @@ func (d *downloadCmd) Run(ctx context.Context, kctx *kong.Context) error {
return err
}
pth, err := config.DownloadDependency(d.Dependency, d.System, &bindown.ConfigDownloadDependencyOpts{
TargetFile: d.TargetFile,
Force: d.Force,
TargetFile: d.TargetFile,
Force: d.Force,
AllowMissingChecksum: d.AllowMissingChecksum,
})
if err != nil {
return err
Expand All @@ -227,9 +232,10 @@ func (d *downloadCmd) Run(ctx context.Context, kctx *kong.Context) error {
}

type extractCmd struct {
System bindown.SystemInfo `kong:"name=system,default=${system_default},help=${system_help},predictor=allSystems"`
Dependency string `kong:"required=true,arg,help=${extract_dependency_help},predictor=bin"`
TargetDir string `kong:"name=output,help=${extract_target_dir_help}"`
System bindown.SystemInfo `kong:"name=system,default=${system_default},help=${system_help},predictor=allSystems"`
Dependency string `kong:"required=true,arg,help=${extract_dependency_help},predictor=bin"`
TargetDir string `kong:"name=output,help=${extract_target_dir_help}"`
AllowMissingChecksum bool `kong:"name=allow-missing-checksum,help=${allow_missing_checksum}"`
}

func (d *extractCmd) Run(ctx context.Context, kctx *kong.Context) error {
Expand All @@ -238,8 +244,9 @@ func (d *extractCmd) Run(ctx context.Context, kctx *kong.Context) error {
return err
}
pth, err := config.ExtractDependency(d.Dependency, d.System, &bindown.ConfigExtractDependencyOpts{
TargetDirectory: d.TargetDir,
Force: false,
TargetDirectory: d.TargetDir,
Force: false,
AllowMissingChecksum: d.AllowMissingChecksum,
})
if err != nil {
return err
Expand Down
98 changes: 82 additions & 16 deletions config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ func (c *Config) addChecksum(dependencyName string, sysInfo SystemInfo) error {
if existingSum != "" {
return nil
}
sum, err := getURLChecksum(*dep.URL)
sum, err := getURLChecksum(*dep.URL, "")
if err != nil {
return err
}
Expand Down Expand Up @@ -334,8 +334,9 @@ func (c *Config) validateDep(systems []SystemInfo, depName string) error {

// ConfigDownloadDependencyOpts options for Config.DownloadDependency
type ConfigDownloadDependencyOpts struct {
TargetFile string
Force bool
TargetFile string
Force bool
AllowMissingChecksum bool
}

// extractsCacheDir returns the cache directory for an extraction based on the download's checksum and dependency name
Expand All @@ -357,7 +358,7 @@ func (c *Config) downloadCacheDir(checksum string) (string, error) {
}

// DownloadDependency downloads a dependency
func (c *Config) DownloadDependency(dependencyName string, sysInfo SystemInfo, opts *ConfigDownloadDependencyOpts) (string, error) {
func (c *Config) DownloadDependency(dependencyName string, sysInfo SystemInfo, opts *ConfigDownloadDependencyOpts) (_ string, errOut error) {
if opts == nil {
opts = &ConfigDownloadDependencyOpts{}
}
Expand All @@ -366,27 +367,81 @@ func (c *Config) DownloadDependency(dependencyName string, sysInfo SystemInfo, o
if err != nil {
return "", err
}
if dep.URL == nil {
return "", fmt.Errorf("no URL configured")
depURL, err := dep.url()
if err != nil {
return "", err
}

checksum, err := c.dependencyChecksum(dependencyName, sysInfo)
tempDir, err := os.MkdirTemp("", "bindown")
if err != nil {
return "", err
}
defer func() {
cleanupErr := os.RemoveAll(tempDir)
if errOut == nil {
errOut = cleanupErr
}
}()
tempFile := filepath.Join(tempDir, "download")
downloadedToTemp := false
checksum, err := c.dependencyChecksum(dependencyName, sysInfo)
if err != nil {
if !opts.AllowMissingChecksum {
return "", err
}
checksum, err = getURLChecksum(depURL, tempFile)
if err != nil {
return "", err
}
downloadedToTemp = true
}

if targetFile == "" {
dlFile, err := urlFilename(*dep.URL)
var dlFile, cacheDir string
dlFile, err = urlFilename(depURL)
if err != nil {
return "", err
}
cacheDir, err := c.downloadCacheDir(checksum)
cacheDir, err = c.downloadCacheDir(checksum)
if err != nil {
return "", err
}
targetFile = filepath.Join(cacheDir, dlFile)
}
return targetFile, download(strFromPtr(dep.URL), targetFile, checksum, opts.Force)

if !downloadedToTemp {
return targetFile, download(depURL, targetFile, checksum, opts.Force)
}

ok, err := fileExistsWithChecksum(targetFile, checksum)
if err != nil {
return "", err
}
if ok {
return targetFile, nil
}

err = os.MkdirAll(filepath.Dir(targetFile), 0o750)
if err != nil {
return "", err
}

// copy the file from the temp dir to the target file
out, err := os.Create(targetFile)
if err != nil {
return "", err
}
defer logCloseErr(out)
in, err := os.Open(tempFile)
if err != nil {
return "", err
}
defer logCloseErr(in)
_, err = io.Copy(out, in)
if err != nil {
return "", err
}
return targetFile, nil
}

func urlFilename(dlURL string) (string, error) {
Expand Down Expand Up @@ -414,8 +469,9 @@ func (c *Config) dependencyChecksum(dependencyName string, sysInfo SystemInfo) (

// ConfigExtractDependencyOpts options for Config.ExtractDependency
type ConfigExtractDependencyOpts struct {
TargetDirectory string
Force bool
TargetDirectory string
Force bool
AllowMissingChecksum bool
}

// ExtractDependency downloads and extracts a dependency
Expand All @@ -424,7 +480,8 @@ func (c *Config) ExtractDependency(dependencyName string, sysInfo SystemInfo, op
opts = &ConfigExtractDependencyOpts{}
}
downloadPath, err := c.DownloadDependency(dependencyName, sysInfo, &ConfigDownloadDependencyOpts{
Force: opts.Force,
Force: opts.Force,
AllowMissingChecksum: opts.AllowMissingChecksum,
})
if err != nil {
return "", err
Expand All @@ -443,7 +500,13 @@ func (c *Config) ExtractDependency(dependencyName string, sysInfo SystemInfo, op
var checksum string
checksum, err = c.dependencyChecksum(dependencyName, sysInfo)
if err != nil {
return "", err
if !opts.AllowMissingChecksum {
return "", err
}
checksum, err = fileChecksum(downloadPath)
if err != nil {
return "", err
}
}
targetDir, err = c.extractsCacheDir(dependencyName, checksum)
if err != nil {
Expand All @@ -465,8 +528,10 @@ func (c *Config) ExtractDependency(dependencyName string, sysInfo SystemInfo, op
type ConfigInstallDependencyOpts struct {
// TargetPath is the path where the executable should end up
TargetPath string
// Force - whether to force the install even if it already exists
// Force - install even if it already exists
Force bool
// AllowMissingChecksum - whether to allow missing checksum
AllowMissingChecksum bool
}

// InstallDependency downloads, extracts and installs a dependency
Expand All @@ -475,7 +540,8 @@ func (c *Config) InstallDependency(dependencyName string, sysInfo SystemInfo, op
opts = &ConfigInstallDependencyOpts{}
}
extractDir, err := c.ExtractDependency(dependencyName, sysInfo, &ConfigExtractDependencyOpts{
Force: opts.Force,
Force: opts.Force,
AllowMissingChecksum: opts.AllowMissingChecksum,
})
if err != nil {
return "", err
Expand Down
40 changes: 26 additions & 14 deletions dependency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,13 @@ func varsWithSubstitutions(vars map[string]string, subs map[string]map[string]st
return vars
}

func (d *Dependency) url() (string, error) {
if d.URL == nil {
return "", fmt.Errorf("no URL configured")
}
return *d.URL, nil
}

func (d *Dependency) clone() *Dependency {
dep := *d
if d.Vars != nil {
Expand Down Expand Up @@ -435,24 +442,29 @@ func extract(archivePath, extractDir string) error {
return os.WriteFile(extractSumFile, []byte(extractSum), 0o600)
}

// getURLChecksum returns the checksum of what is returned from this url
func getURLChecksum(dlURL string) (string, error) {
downloadDir, err := os.MkdirTemp("", "bindown")
if err != nil {
return "", err
// getURLChecksum returns the checksum of the file at dlURL. If tempFile is specified
// it will be used as the temporary file to download the file to and it will be the caller's
// responsibility to clean it up. Otherwise, a temporary file will be created and cleaned up
// automatically.
func getURLChecksum(dlURL, tempFile string) (_ string, errOut error) {
if tempFile == "" {
downloadDir, err := os.MkdirTemp("", "bindown")
if err != nil {
return "", err
}
tempFile = filepath.Join(downloadDir, "foo")
defer func() {
cleanupErr := os.RemoveAll(downloadDir)
if errOut == nil {
errOut = cleanupErr
}
}()
}
dlPath := filepath.Join(downloadDir, "foo")
err = downloadFile(dlPath, dlURL)
err := downloadFile(tempFile, dlURL)
if err != nil {
_ = os.RemoveAll(downloadDir) //nolint:errcheck // we already have an error to report
return "", err
}
sum, err := fileChecksum(dlPath)
cleanupErr := os.RemoveAll(downloadDir)
if err == nil {
err = cleanupErr
}
return sum, err
return fileChecksum(tempFile)
}

func downloadFile(targetPath, url string) error {
Expand Down
2 changes: 1 addition & 1 deletion dependency_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func Test_downloadFile(t *testing.T) {

func TestGetURLChecksum(t *testing.T) {
ts := serveFile(t, filepath.Join("testdata", "downloadables", "foo.tar.gz"), "/foo/foo.tar.gz", "")
got, err := getURLChecksum(ts.URL + "/foo/foo.tar.gz")
got, err := getURLChecksum(ts.URL+"/foo/foo.tar.gz", "")
require.NoError(t, err)
require.Equal(t, fooChecksum, got)
}
Expand Down

0 comments on commit 0a66f04

Please sign in to comment.