feat: initial commit

.gitignore

@@ -0,0 +1,6 @@
+# goland
+.idea/*
+
+# generated artifacts
+.dist/*
+.tmp/*

README.md

@@ -0,0 +1,3 @@
+# Repo Analyzer
+
+> a go library to analyze a project directory to determinate all modules / languages / build-systems used.

analyzer.go

@@ -0,0 +1,69 @@
+package repoanalyzer
+
+import (
+	"github.com/cidverse/repoanalyzer/analyzer/container"
+	"github.com/cidverse/repoanalyzer/analyzer/gomod"
+	"github.com/cidverse/repoanalyzer/analyzer/gradle"
+	"github.com/cidverse/repoanalyzer/analyzer/helm"
+	"github.com/cidverse/repoanalyzer/analyzer/hugo"
+	"github.com/cidverse/repoanalyzer/analyzer/node"
+	"github.com/cidverse/repoanalyzer/analyzer/python"
+	"github.com/cidverse/repoanalyzer/logger"
+	"github.com/thoas/go-funk"
+	"strings"
+	"time"
+
+	"github.com/cidverse/repoanalyzer/analyzerapi"
+)
+
+var analyzerCache = make(map[string][]*analyzerapi.ProjectModule)
+
+// AnalyzeProject will analyze a project and return all modules in path
+func AnalyzeProject(projectDir string, path string) []*analyzerapi.ProjectModule {
+	if funk.Contains(analyzerCache, path) {
+		return analyzerCache[path]
+	}
+
+	if len(analyzerapi.Analyzers) == 0 {
+		initAnalyzers()
+	}
+
+	start := time.Now()
+	logger.Logger.Info("repo analyzer start", "path", path, "scanners", len(analyzerapi.Analyzers))
+
+	// prepare context
+	ctx := analyzerapi.GetAnalyzerContext(projectDir)
+
+	// run
+	var allModules []*analyzerapi.ProjectModule
+	var allModuleNames []string
+	for _, a := range analyzerapi.Analyzers {
+		logger.Debug("repo analyzer run", "name", a.GetName())
+
+		modules := a.Analyze(ctx)
+		for _, module := range modules {
+			currentModule := module
+			if strings.HasPrefix(currentModule.Directory, path) && !strings.Contains(currentModule.Directory, "testdata") {
+				allModules = append(allModules, currentModule)
+				allModuleNames = append(allModuleNames, currentModule.Slug)
+			}
+		}
+	}
+
+	logger.Info("repo analyzer complete", "module_count", len(allModules), "modules", allModuleNames, "duration", time.Since(start).String(), "file_count", len(ctx.Files))
+
+	analyzerCache[projectDir] = allModules
+	return allModules
+}
+
+func initAnalyzers() {
+	analyzerapi.Analyzers = append(analyzerapi.Analyzers,
+		container.Analyzer{},
+		gomod.Analyzer{},
+		gradle.Analyzer{},
+		helm.Analyzer{},
+		hugo.Analyzer{},
+		node.Analyzer{},
+		python.Analyzer{},
+	)
+}

analyzer/container/container.go

@@ -0,0 +1,83 @@
+package container
+
+import (
+	"github.com/cidverse/repoanalyzer/logger"
+	"github.com/cidverse/repoanalyzer/util"
+	"golang.org/x/exp/slices"
+	"path/filepath"
+	"strings"
+
+	"github.com/cidverse/repoanalyzer/analyzerapi"
+	"github.com/gosimple/slug"
+)
+
+type Analyzer struct{}
+
+func (a Analyzer) GetName() string {
+	return "container"
+}
+
+func (a Analyzer) Analyze(ctx analyzerapi.AnalyzerContext) []*analyzerapi.ProjectModule {
+	var result []*analyzerapi.ProjectModule
+
+	// dockerfile
+	for _, file := range ctx.Files {
+		filename := filepath.Base(file)
+
+		if filename == "Dockerfile" || filename == "Containerfile" || strings.HasSuffix(filename, ".Dockerfile") || strings.HasSuffix(filename, ".Containerfile") {
+			// add new module or append file to existing module
+			moduleIdx := slices.IndexFunc(result, func(m *analyzerapi.ProjectModule) bool {
+				return m.Name == filepath.Base(filepath.Dir(file)) && m.BuildSystem == analyzerapi.BuildSystemContainer && m.BuildSystemSyntax == analyzerapi.ContainerFile
+			})
+			if moduleIdx == -1 {
+				module := analyzerapi.ProjectModule{
+					RootDirectory:     ctx.ProjectDir,
+					Directory:         filepath.Dir(file),
+					Name:              filepath.Base(filepath.Dir(file)),
+					Slug:              slug.Make(filepath.Base(filepath.Dir(file))),
+					Discovery:         []string{"file~" + file},
+					BuildSystem:       analyzerapi.BuildSystemContainer,
+					BuildSystemSyntax: analyzerapi.ContainerFile,
+					Language:          nil,
+					Dependencies:      nil,
+					Submodules:        nil,
+					Files:             ctx.Files,
+					FilesByExtension:  ctx.FilesByExtension,
+				}
+				analyzerapi.AddModuleToResult(&result, &module)
+			} else {
+				result[moduleIdx].Discovery = append(result[moduleIdx].Discovery, "file~"+file)
+			}
+		}
+	}
+
+	// buildah
+	for _, file := range ctx.FilesByExtension["sh"] {
+		filename := filepath.Base(file)
+
+		if strings.HasSuffix(filename, ".sh") {
+			content, contentErr := util.GetFileContent(file)
+			if contentErr != nil {
+				logger.Warn("failed to read file content", "file", file)
+			} else if strings.Contains(content, "buildah from") {
+				module := analyzerapi.ProjectModule{
+					RootDirectory:     ctx.ProjectDir,
+					Directory:         filepath.Dir(file),
+					Name:              filepath.Base(filepath.Dir(file)),
+					Slug:              slug.Make(filepath.Base(filepath.Dir(file))),
+					Discovery:         []string{"file~" + file},
+					BuildSystem:       analyzerapi.BuildSystemContainer,
+					BuildSystemSyntax: analyzerapi.ContainerBuildahScript,
+					Language:          nil,
+					Dependencies:      nil,
+					Submodules:        nil,
+					Files:             ctx.Files,
+					FilesByExtension:  ctx.FilesByExtension,
+				}
+				analyzerapi.AddModuleToResult(&result, &module)
+			}
+		}
+	}
+
+	return result
+}

analyzer/gomod/gomod.go

@@ -0,0 +1,71 @@
+package gomod
+
+import (
+	"github.com/cidverse/repoanalyzer/util"
+	"path/filepath"
+
+	"github.com/cidverse/repoanalyzer/analyzerapi"
+	"github.com/gosimple/slug"
+	"golang.org/x/mod/modfile"
+)
+
+type Analyzer struct{}
+
+func (a Analyzer) GetName() string {
+	return "gomod"
+}
+
+func (a Analyzer) Analyze(ctx analyzerapi.AnalyzerContext) []*analyzerapi.ProjectModule {
+	var result []*analyzerapi.ProjectModule
+
+	for _, file := range ctx.FilesByExtension["mod"] {
+		filename := filepath.Base(file)
+
+		// detect build system syntax
+		if filename == "go.mod" {
+			// parse go.mod
+			contentBytes, contentReadErr := util.GetFileBytes(file)
+			if contentReadErr != nil {
+				continue
+			}
+			goMod, goModParseError := modfile.ParseLax(file, contentBytes, nil)
+			if goModParseError != nil {
+				continue
+			}
+
+			// references
+			goVersion := goMod.Go.Version + ".0"
+
+			// deps
+			var dependencies []analyzerapi.ProjectDependency
+			for _, req := range goMod.Require {
+				dep := analyzerapi.ProjectDependency{
+					Type:    string(analyzerapi.BuildSystemGoMod),
+					ID:      req.Mod.Path,
+					Version: req.Mod.Version,
+				}
+				dependencies = append(dependencies, dep)
+			}
+
+			// module
+			module := analyzerapi.ProjectModule{
+				RootDirectory:     ctx.ProjectDir,
+				Directory:         filepath.Dir(file),
+				Name:              goMod.Module.Mod.Path,
+				Slug:              slug.Make(goMod.Module.Mod.Path),
+				Discovery:         []string{"file~" + file},
+				BuildSystem:       analyzerapi.BuildSystemGoMod,
+				BuildSystemSyntax: analyzerapi.BuildSystemSyntaxDefault,
+				Language:          analyzerapi.GetSingleLanguageMap(analyzerapi.LanguageGolang, &goVersion),
+				Dependencies:      dependencies,
+				Submodules:        nil,
+				Files:             ctx.Files,
+				FilesByExtension:  ctx.FilesByExtension,
+			}
+
+			analyzerapi.AddModuleToResult(&result, &module)
+		}
+	}
+
+	return result
+}

analyzer/gomod/gomod_test.go

@@ -0,0 +1,40 @@
+package gomod
+
+import (
+	"github.com/cidverse/repoanalyzer/logger"
+	"github.com/go-logr/logr/testr"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/cidverse/repoanalyzer/analyzerapi"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGoModAnalyzer_Analyze(t *testing.T) {
+	logger.Logger = testr.New(t)
+
+	cwd, err := os.Getwd()
+	assert.NoError(t, err)
+
+	ctx := analyzerapi.GetAnalyzerContext(filepath.Join(filepath.Dir(cwd), "..", "testdata", "gomod"))
+	analyzer := Analyzer{}
+	result := analyzer.Analyze(ctx)
+
+	// module
+	assert.Len(t, result, 1)
+	assert.Equal(t, "github.com/dummymodule", result[0].Name)
+	assert.Equal(t, analyzerapi.BuildSystemGoMod, result[0].BuildSystem)
+	assert.Equal(t, analyzerapi.BuildSystemSyntaxDefault, result[0].BuildSystemSyntax)
+	assert.NotNil(t, result[0].Language[analyzerapi.LanguageGolang])
+	assert.Equal(t, "1.16.0", *result[0].Language[analyzerapi.LanguageGolang])
+	assert.Equal(t, "gomod", result[0].Dependencies[0].Type)
+	assert.Equal(t, "github.com/Masterminds/semver/v3", result[0].Dependencies[0].ID)
+	assert.Equal(t, "v3.1.1", result[0].Dependencies[0].Version)
+
+	// submodule
+	assert.Len(t, result[0].Submodules, 0)
+
+	// print result
+	logger.Info("output", "result", result)
+}

analyzer/gradle/gradle.go

@@ -0,0 +1,58 @@
+package gradle
+
+import (
+	"path/filepath"
+
+	"github.com/cidverse/repoanalyzer/analyzerapi"
+	"github.com/gosimple/slug"
+)
+
+type Analyzer struct{}
+
+func (a Analyzer) GetName() string {
+	return "gradle"
+}
+
+func (a Analyzer) Analyze(ctx analyzerapi.AnalyzerContext) []*analyzerapi.ProjectModule {
+	var result []*analyzerapi.ProjectModule
+
+	for _, file := range ctx.Files {
+		filename := filepath.Base(file)
+
+		// detect build system syntax
+		var buildSystemSyntax analyzerapi.ProjectBuildSystemSyntax
+		if filename == "build.gradle" {
+			buildSystemSyntax = analyzerapi.GradleGroovyDSL
+		} else if filename == "build.gradle.kts" {
+			buildSystemSyntax = analyzerapi.GradleKotlinDSL
+		} else {
+			continue
+		}
+
+		// language
+		language := make(map[analyzerapi.ProjectLanguage]*string)
+		language[analyzerapi.LanguageJava] = nil
+
+		// deps
+		var dependencies []analyzerapi.ProjectDependency
+
+		// module
+		module := analyzerapi.ProjectModule{
+			RootDirectory:     ctx.ProjectDir,
+			Directory:         filepath.Dir(file),
+			Name:              filepath.Base(filepath.Dir(file)),
+			Slug:              slug.Make(filepath.Base(filepath.Dir(file))),
+			Discovery:         []string{"file~" + file},
+			BuildSystem:       analyzerapi.BuildSystemGradle,
+			BuildSystemSyntax: buildSystemSyntax,
+			Language:          language,
+			Dependencies:      dependencies,
+			Submodules:        nil,
+			Files:             ctx.Files,
+			FilesByExtension:  ctx.FilesByExtension,
+		}
+		analyzerapi.AddModuleToResult(&result, &module)
+	}
+
+	return result
+}

analyzer/gradle/gradle_test.go

@@ -0,0 +1,61 @@
+package gradle
+
+import (
+	"github.com/cidverse/repoanalyzer/logger"
+	"github.com/cidverse/repoanalyzer/util"
+	"github.com/go-logr/logr/testr"
+	"testing"
+
+	"github.com/cidverse/repoanalyzer/analyzerapi"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGradleAnalyzer_AnalyzeGroovy(t *testing.T) {
+	logger.Logger = testr.New(t)
+
+	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "gradle-groovy"))
+	analyzer := Analyzer{}
+	result := analyzer.Analyze(ctx)
+
+	// module
+	assert.Len(t, result, 1)
+	assert.Equal(t, "gradle-groovy", result[0].Name)
+	assert.Equal(t, analyzerapi.BuildSystemGradle, result[0].BuildSystem)
+	assert.Equal(t, analyzerapi.GradleGroovyDSL, result[0].BuildSystemSyntax)
+	assert.Nil(t, result[0].Language[analyzerapi.LanguageJava])
+
+	// submodule
+	assert.Len(t, result[0].Submodules, 1)
+	assert.Equal(t, "gradle-groovy-api", result[0].Submodules[0].Name)
+	assert.Equal(t, analyzerapi.BuildSystemGradle, result[0].Submodules[0].BuildSystem)
+	assert.Equal(t, string(analyzerapi.GradleGroovyDSL), string(result[0].Submodules[0].BuildSystemSyntax))
+	assert.Nil(t, result[0].Submodules[0].Language[analyzerapi.LanguageJava])
+
+	// print result
+	logger.Logger.Info("output", "result", result)
+}
+
+func TestGradleAnalyzer_AnalyzeKotlin(t *testing.T) {
+	logger.Logger = testr.New(t)
+
+	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "gradle-kotlin"))
+	analyzer := Analyzer{}
+	result := analyzer.Analyze(ctx)
+
+	// module
+	assert.Len(t, result, 1)
+	assert.Equal(t, "gradle-kotlin", result[0].Name)
+	assert.Equal(t, analyzerapi.BuildSystemGradle, result[0].BuildSystem)
+	assert.Equal(t, string(analyzerapi.GradleKotlinDSL), string(result[0].BuildSystemSyntax))
+	assert.Nil(t, result[0].Language[analyzerapi.LanguageJava])
+
+	// submodule
+	assert.Len(t, result[0].Submodules, 1)
+	assert.Equal(t, "gradle-kotlin-api", result[0].Submodules[0].Name)
+	assert.Equal(t, analyzerapi.BuildSystemGradle, result[0].Submodules[0].BuildSystem)
+	assert.Equal(t, string(analyzerapi.GradleKotlinDSL), string(result[0].Submodules[0].BuildSystemSyntax))
+	assert.Nil(t, result[0].Submodules[0].Language[analyzerapi.LanguageJava])
+
+	// print result
+	logger.Logger.Info("output", "result", result)
+}