Skip to content

Commit

Permalink
git support
Browse files Browse the repository at this point in the history
  • Loading branch information
@xtuc
xtuc committed Jun 8, 2021
1 parent 2749d45 commit c12b5a3
Show file tree
Hide file tree
Showing 16 changed files with 312 additions and 462 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,7 @@ lint:
dev: autoupdate
docker build -t cdnjs-dev -f ./dev/Dockerfile .
docker run -it cdnjs-dev

.PHONY: process-version-sandbox
process-version-sandbox:
docker build -t $@ -f ./docker/process-version/Dockerfile .
108 changes: 34 additions & 74 deletions cmd/checker/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
"path"
"path/filepath"
Expand All @@ -19,6 +18,7 @@ import (
"github.com/cdnjs/tools/packages"
"github.com/cdnjs/tools/sandbox"
"github.com/cdnjs/tools/util"
"github.com/cdnjs/tools/version"

"github.com/pkg/errors"
)
Expand Down Expand Up @@ -67,22 +67,14 @@ func main() {
}
}

// Represents a version of a package,
// which could be a git version, npm version, etc.
type version interface {
Get() string // Get the version as a string.
Download(...interface{}) string // Download a version, returning the download dir.
Clean(string) // Clean a download dir.
}

func processVersion(ctx context.Context, pckg *packages.Package, version npm.Version) (string, error) {
func processVersion(ctx context.Context, pckg *packages.Package, v version.Version) (string, error) {
inDir, outDir, err := sandbox.Setup()
if err != nil {
return outDir, errors.Wrap(err, "failed to setup sandbox")
}
defer os.RemoveAll(inDir)

buff := npm.DownloadTar(ctx, version.Tarball)
buff := version.DownloadTar(ctx, v)

dst, err := os.Create(path.Join(inDir, "new-version.tgz"))
if err != nil {
Expand All @@ -97,7 +89,7 @@ func processVersion(ctx context.Context, pckg *packages.Package, version npm.Ver
return outDir, errors.Wrap(err, "failed to write configuration")
}

name := fmt.Sprintf("%s_%s", *pckg.Name, version.Version)
name := fmt.Sprintf("%s_%s", *pckg.Name, v.Version)
logs, err := sandbox.Run(ctx, name, inDir, outDir)
if err != nil {
return outDir, errors.Wrap(err, "failed to run sandbox")
Expand Down Expand Up @@ -127,63 +119,45 @@ func showFiles(pckgPath string, noPathValidation bool) error {
// autoupdate exists, download latest versions based on source
src := *pckg.Autoupdate.Source

var versions []version.Version

switch src {
case "npm":
{
// get npm versions and sort
versions, _ := npm.GetVersions(ctx, pckg.Autoupdate)
sort.Sort(npm.ByTimeStamp(versions))

// download into temp dir
if len(versions) > 0 {
// print info for first src version
if err := printMostRecentVersion(ctx, pckg, versions[0]); err != nil {
return errors.Wrap(err, "could not print most recent version")
}

// print aggregate info for the few last src versions
if err := printLastVersions(ctx, pckg, versions[1:]); err != nil {
return errors.Wrap(err, "could not print most last versions")
}
} else {
showErr(ctx, "no version found on npm")
}
versions, _ = npm.GetVersions(ctx, pckg.Autoupdate)
sort.Sort(version.ByDate(versions))
}
case "git":
{
// WIP
var err error
// get git versions and sort
versions, err = git.GetVersions(ctx, pckg.Autoupdate)
if err != nil {
return errors.Wrap(err, "failed to retrieve git versions")
}
sort.Sort(version.ByDate(versions))
}
// case "git":
// {
// // make temp dir and clone
// packageGitDir, direrr := ioutil.TempDir("", src)
// util.Check(direrr)
// out, cloneerr := git.Clone(ctx, *pckg.Autoupdate.Target, packageGitDir)
// if cloneerr != nil {
// showErr(ctx, fmt.Sprintf("could not clone repo: %s: %s\n", cloneerr, out))
// return
// }

// // get git versions and sort
// gitVersions, _ := git.GetVersions(ctx, packageGitDir)
// sort.Sort(git.ByTimeStamp(gitVersions))

// // cast to interface
// for _, v := range gitVersions {
// versions = append(versions, v)
// }

// // set download dir
// downloadDir = packageGitDir

// // set err string if no versions
// noVersionsErr = "no tagged version found in git"
// }
default:
{
panic(fmt.Sprintf("unknown autoupdate source: %s", src))
}
}

// download into temp dir
if len(versions) > 0 {
// print info for first src version
if err := printMostRecentVersion(ctx, pckg, versions[0]); err != nil {
return errors.Wrap(err, "could not print most recent version")
}

// print aggregate info for the few last src versions
if err := printLastVersions(ctx, pckg, versions[1:]); err != nil {
return errors.Wrap(err, "could not print most last versions")
}
} else {
showErr(ctx, "no version found on "+src)
}
return nil
}

Expand Down Expand Up @@ -248,7 +222,7 @@ func filewalker(basedir string, files *[]string) filepath.WalkFunc {

// Prints the files of a package version, outputting debug
// messages if no valid files are present.
func printMostRecentVersion(ctx context.Context, p *packages.Package, v npm.Version) error {
func printMostRecentVersion(ctx context.Context, p *packages.Package, v version.Version) error {
fmt.Printf("\nmost recent version: %s\n", v.Version)

outDir, err := processVersion(ctx, p, v)
Expand All @@ -265,7 +239,7 @@ func printMostRecentVersion(ctx context.Context, p *packages.Package, v npm.Vers
}

if len(files) == 0 {
errormsg := fmt.Sprintf("No files will be published for version %s.\n", v.Get())
errormsg := fmt.Sprintf("No files will be published for version %s.\n", v.Version)
showErr(ctx, errormsg)
return nil
}
Expand All @@ -282,15 +256,15 @@ func printMostRecentVersion(ctx context.Context, p *packages.Package, v npm.Vers
fmt.Printf("```\n")

if p.Filename != nil && !filenameFound {
showErr(ctx, fmt.Sprintf("Filename `%s` not found in most recent version `%s`.\n", *p.Filename, v.Get()))
showErr(ctx, fmt.Sprintf("Filename `%s` not found in most recent version `%s`.\n", *p.Filename, v.Version))
}
return nil
}

// Prints the matching files of a number of last versions.
// Each previous version will be downloaded and cleaned up if necessary.
// For example, a temporary directory may be downloaded and then removed later.
func printLastVersions(ctx context.Context, p *packages.Package, versions []npm.Version) error {
func printLastVersions(ctx context.Context, p *packages.Package, versions []version.Version) error {
// limit versions
if len(versions) > util.ImportAllMaxVersions {
versions = versions[:util.ImportAllMaxVersions]
Expand Down Expand Up @@ -322,20 +296,6 @@ func printLastVersions(ctx context.Context, p *packages.Package, versions []npm.
return nil
}

func makeGlobDebugLink(glob string, dir string) string {
encodedGlob := url.QueryEscape(glob)
allTests := ""

util.Check(filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
allTests += "&tests=" + url.QueryEscape(info.Name())
}
return nil
}))

return fmt.Sprintf("https://www.digitalocean.com/community/tools/glob?comments=true&glob=%s&matches=true%s&tests=", encodedGlob, allTests)
}

func checkGitHubPopularity(ctx context.Context, pckg *packages.Package) bool {
if !strings.Contains(*pckg.Repository.URL, "github.com") {
return false
Expand Down
24 changes: 18 additions & 6 deletions cmd/process-version/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func main() {
log.Fatalf("could not create workspace: %s", err)
}

if err := extractInput(); err != nil {
if err := extractInput(*config.Autoupdate.Source); err != nil {
log.Fatalf("failed to extract input: %s", err)
}

Expand Down Expand Up @@ -124,6 +124,11 @@ func removePackageDir(path string) string {
return path
}

func removeFirstDir(path string) string {
parts := strings.Split(path, "/")
return strings.Replace(path, parts[0]+"/", "", 1)
}

func readConfig() (*packages.Package, error) {
file := path.Join(INPUT, "config.json")
data, err := ioutil.ReadFile(file)
Expand All @@ -137,7 +142,7 @@ func readConfig() (*packages.Package, error) {
return config, nil
}

func extractInput() error {
func extractInput(source string) error {
gzipStream, err := os.Open(path.Join(INPUT, "new-version.tgz"))
if err != nil {
return errors.Wrap(err, "could not open input")
Expand All @@ -159,8 +164,15 @@ func extractInput() error {
log.Fatalf("ExtractTarGz: Next() failed: %s", err.Error())
}

// remove package folder
target := removePackageDir(header.Name)
target := header.Name
if source == "npm" {
// remove package folder
target = removePackageDir(header.Name)
}
if source == "git" {
// remove package folder
target = removeFirstDir(header.Name)
}

switch header.Typeflag {
case tar.TypeDir:
Expand All @@ -178,8 +190,8 @@ func extractInput() error {
return errors.Wrap(err, "ExtractTarGz: Copy() failed")
}
default:
return errors.Errorf(
"ExtractTarGz: uknown type: %x in %s",
log.Printf(
"ExtractTarGz: uknown type: %x in %s\n",
header.Typeflag,
header.Name)
}
Expand Down
41 changes: 0 additions & 41 deletions functions/check-pkg-updates/git.go
View file

This file was deleted.

32 changes: 19 additions & 13 deletions functions/check-pkg-updates/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ import (
)

var (
KV_TOKEN = os.Getenv("KV_TOKEN")
CF_ACCOUNT_ID = os.Getenv("CF_ACCOUNT_ID")
RESTRICT_PKGS = strings.Split(os.Getenv("RESTRICT_PKGS"), ",")
KV_TOKEN = os.Getenv("KV_TOKEN")
CF_ACCOUNT_ID = os.Getenv("CF_ACCOUNT_ID")
PKG_AUTOUPDATE_SOURCE = os.Getenv("PKG_AUTOUPDATE_SOURCE")
RESTRICT_PKGS = strings.Split(os.Getenv("RESTRICT_PKGS"), ",")
)

type APIPackage struct {
Expand All @@ -46,6 +47,10 @@ func Invoke(w http.ResponseWriter, r *http.Request) {
sentry.Init()
defer sentry.PanicHandler()

if PKG_AUTOUPDATE_SOURCE == "" {
panic("PKG_AUTOUPDATE_SOURCE should be present")
}

list, err := packages.FetchPackages()
if err != nil {
http.Error(w, "failed to fetch packages", 500)
Expand Down Expand Up @@ -91,22 +96,23 @@ func checkPackage(pkg *packages.Package) error {
return nil
}

switch *pkg.Autoupdate.Source {
src := *pkg.Autoupdate.Source
if src != PKG_AUTOUPDATE_SOURCE {
// we are not auto-updateing packages with that source; skip.
return nil
}

switch src {
case "npm":
case "git":
{
if err := updateNpm(ctx, pkg); err != nil {
return errors.Wrap(err, "failed to update package via npm")
if err := updatePackage(ctx, pkg); err != nil {
return errors.Wrap(err, "failed to update package via "+src)
}
}
// case "git":
// {
// if err := updateGit(ctx, pkg); err != nil {
// return errors.Wrap(err, "failed to update package via git")
// }
// }
default:
{
return errors.Errorf("%s invalid autoupdate source: %s", *pkg.Name, *pkg.Autoupdate.Source)
return errors.Errorf("%s invalid autoupdate source: %s", *pkg.Name, src)
}
}
return nil
Expand Down
Loading

0 comments on commit c12b5a3

Please sign in to comment.