Merge pull request #76 from projectdiscovery/dev

v0.0.8

README.md

@@ -41,6 +41,7 @@
   - **[FOFA](https://fofa.info)**
   - **[Hunter](https://hunter.qianxin.com)**
   - **[Quake](https://quake.360.net/quake/#/index)**
+  - **[Zoomeye](https://www.zoomeye.org)**
 - Multiple API key input support
 - Automatic API key randomization
 - **stdin** / **stdout** support for input
@@ -68,7 +69,7 @@ Usage:
 Flags:
 INPUT:
    -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
-   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter) (default shodan)
+   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye) (default shodan)
 
 SEARCH-ENGINE:
    -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')
@@ -77,6 +78,7 @@ SEARCH-ENGINE:
    -cs, -censys string[]      search query for censys (example: -censys 'query.txt')
    -qk, -quake string[]       search query for quake (example: -quake 'query.txt')
    -ht, -hunter string[]      search query for hunter (example: -hunter 'query.txt')
+   -ze, -zoomeye string[]     search query for zoomeye (example: -zoomeye 'query.txt')
 
 CONFIG:
    -pc, -provider string  provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
@@ -122,6 +124,9 @@ quake:
 hunter:
   - HUNTER_API_KEY_1
   - HUNTER_API_KEY_2
+zoomeye:
+  - ZOOMEYE_API_KEY_1
+  - ZOOMEYE_API_KEY_2
 ```
 
 When multiple keys/credentials are specified for same provider in the config file, random key will be used for each execution.
@@ -134,10 +139,12 @@ export CENSYS_API_ID=xxx
 export CENSYS_API_SECRET=xxx
 export FOFA_EMAIL=xxx
 export FOFA_KEY=xxx
+export QUAKE_TOKEN=xxx
 export HUNTER_API_KEY=xxx
+export ZOOMEYE_API_KEY=xxx
 ```
 
-Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index) and [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1) .
+Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1) and [ZoomEye](https://www.zoomeye.org/login) .
 
 ## Running Uncover
 
@@ -216,7 +223,7 @@ uncover -q dorks.txt
 **uncover** supports multiple search engine, as default **shodan** is used, `-e` flag can be used to run same query against any or all search engines.
 
 ```console
-echo jira | uncover -e shodan,censys,fofa,quake,hunter
+echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye
 
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
@@ -248,7 +255,7 @@ echo jira | uncover -e shodan,censys,fofa,quake,hunter
 
 
 ```console
-uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira'
+uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira' -zoomeye 'app:"Atlassian JIRA"'
 
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/

runner/banners.go

@@ -8,11 +8,11 @@ const banner = `
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.7
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.8
 `
 
 // Version is the current version of uncover
-const Version = `v0.0.7`
+const Version = `v0.0.8`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

runner/options.go

@@ -47,6 +47,7 @@ type Options struct {
 	Censys       goflags.StringSlice
 	Quake        goflags.StringSlice
 	Hunter       goflags.StringSlice
+	ZoomEye      goflags.StringSlice
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -58,7 +59,7 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
-		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye) (default shodan)", goflags.FileNormalizedStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("search-engine", "Search-Engine",
@@ -68,6 +69,7 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.Censys, "censys", "cs", nil, "search query for censys (example: -censys 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Quake, "quake", "qk", nil, "search query for quake (example: -quake 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Hunter, "hunter", "ht", nil, "search query for hunter (example: -hunter 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ZoomEye, "zoomeye", "ze", nil, "search query for zoomeye (example: -zoomeye 'query.txt')", goflags.FileStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("config", "Config",
@@ -124,7 +126,7 @@ func ParseOptions() *Options {
 		gologger.Warning().Msgf("couldn't parse env vars: %s\n", err)
 	}
 
-	if len(options.Engine) == 0 {
+	if len(options.Engine) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 && len(options.ZoomEye) == 0 {
 		options.Engine = append(options.Engine, "shodan")
 		options.Engine = append(options.Engine, "shodan-idb")
 	}
@@ -193,14 +195,20 @@ func (options *Options) loadProvidersFromEnv() error {
 	if key, exists := os.LookupEnv("HUNTER_API_KEY"); exists {
 		options.Provider.Hunter = append(options.Provider.Hunter, key)
 	}
+	if key, exists := os.LookupEnv("QUAKE_TOKEN"); exists {
+		options.Provider.Quake = append(options.Provider.Quake, key)
+	}
+	if key, exists := os.LookupEnv("ZOOMEYE_API_KEY"); exists {
+		options.Provider.ZoomEye = append(options.Provider.ZoomEye, key)
+	}
 	return nil
 }
 
 // validateOptions validates the configuration options passed
 func (options *Options) validateOptions() error {
 	// Check if domain, list of domains, or stdin info was provided.
 	// If none was provided, then return.
-	if len(options.Query) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 {
+	if len(options.Query) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 && len(options.ZoomEye) == 0 {
 		return errors.New("no query provided")
 	}
 
@@ -210,7 +218,7 @@ func (options *Options) validateOptions() error {
 	}
 
 	// Validate threads and options
-	if len(options.Engine) == 0 {
+	if len(options.Engine) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 && len(options.ZoomEye) == 0 {
 		return errors.New("no engine specified")
 	}
 

runner/output_writer.go

@@ -11,11 +11,16 @@ import (
 )
 
 type OutputWriter struct {
-	cache   *lru.Cache
-	writers []io.Writer
+	cache        *lru.Cache
+	namedWriters []NamedWriter
 	sync.RWMutex
 }
 
+type NamedWriter struct {
+	Writer io.Writer
+	Name   string
+}
+
 func NewOutputWriter() (*OutputWriter, error) {
 	lastPrintedCache, err := lru.New(2048)
 	if err != nil {
@@ -24,19 +29,36 @@ func NewOutputWriter() (*OutputWriter, error) {
 	return &OutputWriter{cache: lastPrintedCache}, nil
 }
 
-func (o *OutputWriter) AddWriters(writers ...io.Writer) {
-	o.writers = append(o.writers, writers...)
+func (o *OutputWriter) AddWriters(named ...NamedWriter) {
+	o.namedWriters = append(o.namedWriters, named...)
+}
+
+// WriteAll writes the data taken as input using
+// all the writers.
+func (o *OutputWriter) WriteAll(data []byte) {
+	o.Lock()
+	defer o.Unlock()
+
+	for _, w := range o.namedWriters {
+		_, _ = w.Writer.Write(data)
+		_, _ = w.Writer.Write([]byte("\n"))
+	}
 }
 
-func (o *OutputWriter) Write(data []byte) {
+// Write writes the data taken as input using only
+// the writer(s) with that name.
+func (o *OutputWriter) Write(name string, data []byte) {
 	o.Lock()
 	defer o.Unlock()
 
-	for _, writer := range o.writers {
-		_, _ = writer.Write(data)
-		_, _ = writer.Write([]byte("\n"))
+	for _, w := range o.namedWriters {
+		if w.Name == name {
+			_, _ = w.Writer.Write(data)
+			_, _ = w.Writer.Write([]byte("\n"))
+		}
 	}
 }
+
 func (o *OutputWriter) findDuplicate(data string) bool {
 	// check if we've already printed this data
 	itemHash := sha1.Sum([]byte(data))
@@ -47,15 +69,30 @@ func (o *OutputWriter) findDuplicate(data string) bool {
 	return false
 }
 
-func (o *OutputWriter) WriteString(data string) {
+// WriteString writes the string taken as input using only
+// the writer(s) with that name.
+// If name is empty it writes using all the writers.
+func (o *OutputWriter) WriteString(name string, data string) {
 	if o.findDuplicate(data) {
 		return
 	}
-	o.Write([]byte(data))
+	if name != "" {
+		o.Write(name, []byte(data))
+		return
+	}
+	o.WriteAll([]byte(data))
 }
-func (o *OutputWriter) WriteJsonData(data uncover.Result) {
+
+// WriteJsonData writes the result taken as input in JSON format
+// using only the writer(s) with that name.
+// If name is empty it writes using all the writers.
+func (o *OutputWriter) WriteJsonData(name string, data uncover.Result) {
 	if o.findDuplicate(fmt.Sprintf("%s:%d", data.IP, data.Port)) {
 		return
 	}
-	o.Write([]byte(data.JSON()))
+	if name != "" {
+		o.Write(name, []byte(data.JSON()))
+		return
+	}
+	o.WriteAll([]byte(data.JSON()))
 }

runner/provider.go

@@ -8,11 +8,12 @@ import (
 )
 
 type Provider struct {
-	Shodan []string `yaml:"shodan"`
-	Censys []string `yaml:"censys"`
-	Fofa   []string `yaml:"fofa"`
-	Quake  []string `yaml:"quake"`
-	Hunter []string `yaml:"hunter"`
+	Shodan  []string `yaml:"shodan"`
+	Censys  []string `yaml:"censys"`
+	Fofa    []string `yaml:"fofa"`
+	Quake   []string `yaml:"quake"`
+	Hunter  []string `yaml:"hunter"`
+	ZoomEye []string `yaml:"zoomeye"`
 }
 
 func (provider *Provider) GetKeys() uncover.Keys {
@@ -48,9 +49,13 @@ func (provider *Provider) GetKeys() uncover.Keys {
 		keys.HunterToken = provider.Hunter[rand.Intn(len(provider.Hunter))]
 	}
 
+	if len(provider.ZoomEye) > 0 {
+		keys.ZoomEyeToken = provider.ZoomEye[rand.Intn(len(provider.ZoomEye))]
+	}
+
 	return keys
 }
 
 func (provider *Provider) HasKeys() bool {
-	return len(provider.Censys) > 0 || len(provider.Shodan) > 0 || len(provider.Fofa) > 0 || len(provider.Quake) > 0 || len(provider.Hunter) > 0
+	return len(provider.Censys) > 0 || len(provider.Shodan) > 0 || len(provider.Fofa) > 0 || len(provider.Quake) > 0 || len(provider.Hunter) > 0 || len(provider.ZoomEye) > 0
 }

runner/runner.go

@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/uncover/uncover/agent/quake"
 	"github.com/projectdiscovery/uncover/uncover/agent/shodan"
 	"github.com/projectdiscovery/uncover/uncover/agent/shodanidb"
+	"github.com/projectdiscovery/uncover/uncover/agent/zoomeye"
 	"go.uber.org/ratelimit"
 )
 
@@ -47,21 +48,23 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		return errors.New("no keys provided")
 	}
 
-	var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter ratelimit.Limiter
+	var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter ratelimit.Limiter
 	if r.options.Delay > 0 {
 		censysRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
 		fofaRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
 		shodanRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
 		shodanIdbRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
 		quakeRatelimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
 		hunterRatelimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
+		zoomeyeRatelimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
 	} else {
 		censysRateLimiter = ratelimit.NewUnlimited()
 		fofaRateLimiter = ratelimit.NewUnlimited()
 		shodanRateLimiter = ratelimit.NewUnlimited()
 		shodanIdbRateLimiter = ratelimit.NewUnlimited()
 		quakeRatelimiter = ratelimit.NewUnlimited()
 		hunterRatelimiter = ratelimit.NewUnlimited()
+		zoomeyeRatelimiter = ratelimit.NewUnlimited()
 	}
 
 	var agents []uncover.Agent
@@ -89,6 +92,10 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		r.options.Engine = append(r.options.Engine, "hunter")
 		query = append(query, r.options.Hunter...)
 	}
+	if len(r.options.ZoomEye) > 0 {
+		r.options.Engine = append(r.options.Engine, "zoomeye")
+		query = append(query, r.options.ZoomEye...)
+	}
 
 	// declare clients
 	for _, engine := range r.options.Engine {
@@ -109,6 +116,8 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 			agent, err = quake.NewWithOptions(&uncover.AgentOptions{RateLimiter: quakeRatelimiter})
 		case "hunter":
 			agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterRatelimiter})
+		case "zoomeye":
+			agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: zoomeyeRatelimiter})
 		default:
 			err = errors.New("unknown agent type")
 		}
@@ -118,19 +127,29 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		agents = append(agents, agent)
 	}
 
+	const (
+		stdoutWriterName = "stdout"
+		fileWriterName   = "file"
+	)
+
 	// open the output file - always overwrite
 	outputWriter, err := NewOutputWriter()
 	if err != nil {
 		return err
 	}
-	outputWriter.AddWriters(os.Stdout)
+
+	writerName := stdoutWriterName
+	outputWriter.AddWriters(NamedWriter{os.Stdout, stdoutWriterName})
 	if r.options.OutputFile != "" {
 		outputFile, err := os.Create(r.options.OutputFile)
 		if err != nil {
 			return err
 		}
 		defer outputFile.Close()
-		outputWriter.AddWriters(outputFile)
+		outputWriter.AddWriters(NamedWriter{outputFile, fileWriterName})
+	}
+	if r.options.Verbose {
+		writerName = fileWriterName
 	}
 
 	// enumerate
@@ -170,10 +189,10 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 						gologger.Warning().Label(agent.Name()).Msgf("%s\n", result.Error.Error())
 					case r.options.JSON:
 						gologger.Verbose().Label(agent.Name()).Msgf("%s\n", result.JSON())
-						outputWriter.WriteJsonData(result)
+						outputWriter.WriteJsonData(writerName, result)
 					case r.options.Raw:
 						gologger.Verbose().Label(agent.Name()).Msgf("%s\n", result.RawData())
-						outputWriter.WriteString(result.RawData())
+						outputWriter.WriteString(writerName, result.RawData())
 					default:
 						port := fmt.Sprint(result.Port)
 						replacer := strings.NewReplacer(
@@ -183,13 +202,13 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 						)
 						outData := replacer.Replace(r.options.OutputFields)
 						searchFor := []string{result.IP, port}
-						if result.Host != "" {
+						if result.Host != "" || r.options.OutputFile != "" {
 							searchFor = append(searchFor, result.Host)
 						}
 						// send to output if any of the field got replaced
 						if stringsutil.ContainsAny(outData, searchFor...) {
 							gologger.Verbose().Label(agent.Name()).Msgf("%s\n", outData)
-							outputWriter.WriteString(outData)
+							outputWriter.WriteString(writerName, outData)
 						}
 					}