linker: impl deep model

cmd/root.go

@@ -1,4 +1,4 @@
-// Copyright © 2018 NAME HERE <EMAIL ADDRESS>
+// Copyright © 2018 scnace [email protected]
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ import (
 	"github.com/scbizu/mew/drawer"
 	"github.com/scbizu/mew/filter"
 	"github.com/scbizu/mew/linker"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -33,6 +34,7 @@ var grep string
 var excludeDirs []string
 var isShowJSON bool
 var dumpGraph string
+var deepMode bool
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
@@ -41,30 +43,69 @@ var RootCmd = &cobra.Command{
 	Long:  `mew - Show your Go repo related pkgs`,
 	Run: func(cmd *cobra.Command, args []string) {
 		l := linker.NewLinker(gopath, repoName)
-		pkgs, err := l.GetAllPkgNames(false, excludeDirs)
-		if err != nil {
-			log.Fatalln(err)
-		}
-		pkgFilter := filter.NewFilter(pkgs)
-		pkgs = pkgFilter.Grep(grep)
-
-		if err = drawer.DrawWithSliceAndSave(dumpGraph, repoName, pkgs); err != nil {
-			log.Fatalln(err.Error())
-		}
-
-		if isShowJSON {
-			jsonRes, err := json.Marshal(pkgs)
-			if err != nil {
-				log.Fatalln(err)
+		if deepMode {
+			if jsonRes := handlePKGMap(l, excludeDirs, grep, dumpGraph, isShowJSON); jsonRes != "" {
+				fmt.Println(jsonRes)
 			}
-			fmt.Println(string(jsonRes))
 		} else {
-			fmt.Println(pkgs)
+			if jsonRes := handlePKGSlice(l, excludeDirs, grep, dumpGraph, repoName, isShowJSON); jsonRes != "" {
+				fmt.Println(jsonRes)
+			}
 		}
+
 		return
 	},
 }
 
+func handlePKGMap(l *linker.Linker, excludeDirs []string, grep string, graphName string, isShowJSON bool) string {
+	var pkgMap map[string][]string
+	var err error
+	pkgMap, err = l.GetAllPKGNames(false, excludeDirs)
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+
+	pkgMapFilter := filter.NewMapFilter(pkgMap)
+	pkgMapFilter.Grep(grep)
+
+	if err = drawer.DrawWithMapAndSave(graphName, pkgMap); err != nil {
+		logrus.Fatalln(err.Error())
+	}
+
+	if isShowJSON {
+		jsonRes, err := json.Marshal(pkgMap)
+		if err != nil {
+			logrus.Fatalln(err)
+		}
+		return string(jsonRes)
+	}
+	return ""
+}
+
+func handlePKGSlice(l *linker.Linker, excludeDirs []string, grep string, graphName string, repo string, isShowJSON bool) string {
+	var pkgs []string
+	var err error
+	pkgs, err = l.GetLayerPKGNames(false, excludeDirs)
+	if err != nil {
+		logrus.Fatalln(err)
+	}
+	pkgFilter := filter.NewFilter(pkgs)
+	pkgs = pkgFilter.Grep(grep)
+
+	if err = drawer.DrawWithSliceAndSave(graphName, repo, pkgs); err != nil {
+		logrus.Fatalln(err.Error())
+	}
+
+	if isShowJSON {
+		jsonRes, err := json.Marshal(pkgs)
+		if err != nil {
+			logrus.Fatalln(err)
+		}
+		return string(jsonRes)
+	}
+	return ""
+}
+
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
@@ -86,5 +127,6 @@ func init() {
 	RootCmd.Flags().StringVarP(&grep, "grep", "g", "", "grep the pkg list")
 	RootCmd.Flags().StringArrayVarP(&excludeDirs, "ed", "e", []string{"vendor", ".git"}, "exclude the dir")
 	RootCmd.Flags().BoolVar(&isShowJSON, "json", false, "show json format")
+	RootCmd.Flags().BoolVar(&deepMode, "deep", false, "[Experimental feature]in deep mode,you will get all(include really all dependency) third party related pkg name")
 	RootCmd.Flags().StringVarP(&dumpGraph, "graph", "d", drawer.DefaultFileName, "dump graphviz graph")
 }

drawer/drawer.go

@@ -11,23 +11,25 @@ import (
 const (
 	// DefaultFileName defines default file name
 	DefaultFileName = "mew.dot"
+	// GraphName defines parent graph name
+	GraphName = "Mew"
 )
 
-// DrawWithSlice returns the DOT tree
+// DrawWithSlice returns the DOT lang  of a slice
 func DrawWithSlice(baseNode string, pkgNames []string) (string, error) {
 	g := gographviz.NewGraph()
-	g.SetName("G")
+	g.SetName(GraphName)
 	g.SetDir(true)
 	paresedNodeName := string(parseBaseNodeName([]byte(baseNode)))
-	g.AddNode("G", paresedNodeName, nil)
+	g.AddNode(GraphName, paresedNodeName, nil)
 	for _, name := range pkgNames {
-		g.AddNode("G", name, nil)
+		g.AddNode(GraphName, name, nil)
 		g.AddEdge(paresedNodeName, name, true, nil)
 	}
 	return g.String(), nil
 }
 
-// DrawWithSliceAndSave draws the grpah and save
+// DrawWithSliceAndSave draws the grpah and save to current path by a slice
 func DrawWithSliceAndSave(filename string, baseNode string, pkgNames []string) error {
 	dotTree, err := DrawWithSlice(baseNode, pkgNames)
 	if err != nil {
@@ -39,6 +41,33 @@ func DrawWithSliceAndSave(filename string, baseNode string, pkgNames []string) e
 	return nil
 }
 
+// DrawWithMap returns DOT lang of a map
+func DrawWithMap(pkgMap map[string][]string) (string, error) {
+	g := gographviz.NewGraph()
+	g.SetName(GraphName)
+	g.SetDir(true)
+	for repo, pkgs := range pkgMap {
+		paresedNodeName := string(parseBaseNodeName([]byte(repo)))
+		for _, pkg := range pkgs {
+			g.AddNode(GraphName, pkg, nil)
+			g.AddEdge(paresedNodeName, pkg, true, nil)
+		}
+	}
+	return g.String(), nil
+}
+
+// DrawWithMapAndSave draws the graph and save to current path by a map
+func DrawWithMapAndSave(filename string, pkgMap map[string][]string) error {
+	dotTree, err := DrawWithMap(pkgMap)
+	if err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(filename, []byte(dotTree), os.ModePerm); err != nil {
+		return err
+	}
+	return nil
+}
+
 func parseBaseNodeName(baseNode []byte) []byte {
 	bs := []byte{'"'}
 	for _, b := range baseNode {

filter/filter.go

@@ -7,13 +7,25 @@ type Filter struct {
 	PkgNames []string
 }
 
+// MapFilter defines the pkg map filter
+type MapFilter struct {
+	PKGMap map[string][]string
+}
+
 // NewFilter init the filter
 func NewFilter(pkgNames []string) *Filter {
 	return &Filter{
 		PkgNames: pkgNames,
 	}
 }
 
+// NewMapFilter init the map filter
+func NewMapFilter(pkgmap map[string][]string) *MapFilter {
+	return &MapFilter{
+		PKGMap: pkgmap,
+	}
+}
+
 // Grep greps the pkg list
 func (f *Filter) Grep(name string) (pkgNames []string) {
 	for _, pn := range f.PkgNames {
@@ -23,3 +35,14 @@ func (f *Filter) Grep(name string) (pkgNames []string) {
 	}
 	return
 }
+
+// Grep greps the pkg list
+func (f *MapFilter) Grep(name string) map[string][]string {
+	res := make(map[string][]string)
+	for repo, pkgs := range f.PKGMap {
+		f := NewFilter(pkgs)
+		pkgs = f.Grep(name)
+		res[repo] = pkgs
+	}
+	return res
+}

linker/linker.go

@@ -6,6 +6,7 @@ import (
 	"go/token"
 	"io/ioutil"
 	"os"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 )
@@ -20,12 +21,18 @@ func init() {
 	initLogLevel()
 }
 
+var (
+	scannedRepo = make(map[string]bool)
+	pkgMap      = make(map[string][]string)
+)
+
 // NewLinker inits the linker
 func NewLinker(gopath string, repoRootPath string) *Linker {
-	return &Linker{
+	l := &Linker{
 		GoPath:   gopath,
 		RepoPath: repoRootPath,
 	}
+	return l
 }
 
 func initLogLevel() {
@@ -37,8 +44,38 @@ func SetLinkerLogLevel(lv logrus.Level) {
 	logrus.SetLevel(lv)
 }
 
-// GetAllPkgNames get repo related pkgs
-func (l *Linker) GetAllPkgNames(allowDup bool, excludeDirs []string) (pkgNames []string, err error) {
+// GetAllPKGNames gets the full layers packages names
+func (l *Linker) GetAllPKGNames(allowDup bool, excludeDirs []string) (map[string][]string, error) {
+
+	names, err := l.GetLayerPKGNames(allowDup, excludeDirs)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Infof("pkgnames:[%v]", names)
+	for _, repo := range names {
+		if !isThirdPartyPackage(repo) {
+			continue
+		}
+		if _, ok := scannedRepo[repo]; ok {
+			logrus.Warnf("scanned repo:[%v]", repo)
+			continue
+		}
+		scannedRepo[repo] = true
+		logrus.Infof("repo path:%v,package:%v", l.RepoPath, repo)
+		pkgMap[l.RepoPath] = append(pkgMap[l.RepoPath], repo)
+		lk := NewLinker(l.GoPath, repo)
+		pkgMap, err = lk.GetAllPKGNames(allowDup, excludeDirs)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return pkgMap, nil
+}
+
+// GetLayerPKGNames gets the layer(depends on the repo) package names
+// DO NOT SUPPORT GOROOT ENV
+func (l *Linker) GetLayerPKGNames(allowDup bool, excludeDirs []string) (pkgNames []string, err error) {
 	fset := token.NewFileSet()
 	fpath := fmt.Sprintf("%s/src/%s/", l.GoPath, l.RepoPath)
 	allDirs := []string{fpath}
@@ -58,7 +95,7 @@ func (l *Linker) GetAllPkgNames(allowDup bool, excludeDirs []string) (pkgNames [
 			for fk, file := range f.Files {
 				for _, s := range file.Imports {
 					logrus.WithField("file", fk).Debugf("Get PKG:%s", s.Path.Value)
-					pkgNames = append(pkgNames, s.Path.Value)
+					pkgNames = append(pkgNames, trimQuotation(s.Path.Value))
 				}
 			}
 		}
@@ -67,10 +104,28 @@ func (l *Linker) GetAllPkgNames(allowDup bool, excludeDirs []string) (pkgNames [
 		// remove duplicate elem
 		pkgNames = removeDupPkgNames(pkgNames)
 	}
+
 	return
 }
 
+func trimQuotation(pkgName string) string {
+	pkgName = strings.Replace(pkgName, `"`, ``, -1)
+	return pkgName
+}
+
+func isThirdPartyPackage(repo string) bool {
+	tpPrefixes := []string{"github.com"}
+	for _, p := range tpPrefixes {
+		ok := strings.HasPrefix(repo, p)
+		if ok {
+			return true
+		}
+	}
+	return false
+}
+
 // GetInvokeSrcMap get pkg names(as value) with his import file(as key)
+// TODO
 func (l *Linker) GetInvokeSrcMap() (map[string][]string, error) {
 	return nil, nil
 }

mew.dot

@@ -1,18 +1 @@
-digraph G {
-	"github.com/scbizu/mew"->"github.com/scbizu/mew/cmd";
-	"github.com/scbizu/mew"->"github.com/scbizu/mew/drawer";
-	"github.com/scbizu/mew"->"github.com/scbizu/mew/filter";
-	"github.com/scbizu/mew"->"github.com/scbizu/mew/linker";
-	"github.com/scbizu/mew"->"github.com/spf13/cobra";
-	"github.com/scbizu/mew"->"github.com/awalterschulze/gographviz";
-	"github.com/scbizu/mew"->"github.com/sirupsen/logrus";
-	"github.com/awalterschulze/gographviz";
-	"github.com/scbizu/mew";
-	"github.com/scbizu/mew/cmd";
-	"github.com/scbizu/mew/drawer";
-	"github.com/scbizu/mew/filter";
-	"github.com/scbizu/mew/linker";
-	"github.com/sirupsen/logrus";
-	"github.com/spf13/cobra";
-
-}
+error: "github.com/scbizu/mew/drawer" is not a node or a subgraph