Merge pull request #19 from projectdiscovery/dev

v0.0.3

Dockerfile

@@ -1,7 +1,7 @@
-FROM golang:1.18rc1-alpine3.14 AS build-env
+FROM golang:1.18.0-alpine3.14 AS build-env
 RUN go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest
 
-FROM alpine:3.15.0
+FROM alpine:3.15.1
 RUN apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/uncover /usr/local/bin/uncover
 ENTRYPOINT ["uncover"]

README.md

@@ -67,7 +67,8 @@ INPUT:
 CONFIG:
    -pc, -provider string  provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
    -config string         flag configuration file (default "$HOME/.config/uncover/config.yaml")
-   -timeout int           timeout in seconds (default 10)
+   -timeout int           timeout in seconds (default 30)
+   -delay int             delay between requests in seconds (0 to disable) (default 1)
 
 OUTPUT:
    -o, -output string  output file to write found results

go.mod

@@ -9,11 +9,12 @@ require (
 	github.com/projectdiscovery/folderutil v0.0.0-20220212074351-38f1c1d2fdd4
 	github.com/projectdiscovery/goflags v0.0.8-0.20220304165250-2530b305a4a9
 	github.com/projectdiscovery/gologger v1.1.4
-	github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea
 	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
+	go.uber.org/ratelimit v0.2.0
 )
 
 require (
+	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/json-iterator/go v1.1.10 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
+github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
@@ -40,14 +42,17 @@ github.com/projectdiscovery/goflags v0.0.8-0.20220304165250-2530b305a4a9 h1:J05G
 github.com/projectdiscovery/goflags v0.0.8-0.20220304165250-2530b305a4a9/go.mod h1:37KhVbVLllyuIAgpXGqcvE/hsFEwJ+ctEUSHawjhsBY=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
-github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA=
-github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
 github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
+go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210419170143-37df388d1f33 h1:zah5VTTvBlVRELjcDwGLLaWRHZJQsBtplweVYCii0KM=
 golang.org/x/sys v0.0.0-20210419170143-37df388d1f33/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

runner/banners.go

@@ -8,11 +8,11 @@ const banner = `
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.2  
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.3
 `
 
 // Version is the current version of uncover
-const Version = `v0.0.2`
+const Version = `v0.0.3`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

runner/options.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/fileutil"
@@ -12,7 +13,6 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
 	"github.com/projectdiscovery/gologger/levels"
-	"github.com/projectdiscovery/sliceutil"
 )
 
 var (
@@ -35,6 +35,8 @@ type Options struct {
 	Verbose      bool
 	NoColor      bool
 	Timeout      int
+	Delay        int
+	delay        time.Duration
 	Provider     *Provider
 }
 
@@ -53,7 +55,8 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("config", "Config",
 		flagSet.StringVarP(&options.ProviderFile, "provider", "pc", defaultProviderConfigLocation, "provider configuration file"),
 		flagSet.StringVar(&options.ConfigFile, "config", defaultConfigLocation, "flag configuration file"),
-		flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
+		flagSet.IntVar(&options.Timeout, "timeout", 30, "timeout in seconds"),
+		flagSet.IntVar(&options.Delay, "delay", 1, "delay between requests in seconds (0 to disable)"),
 	)
 
 	flagSet.CreateGroup("output", "Output",
@@ -74,7 +77,7 @@ func ParseOptions() *Options {
 		fmt.Println(err.Error())
 		os.Exit(1)
 	}
-	
+
 	options.configureOutput()
 
 	showBanner()
@@ -187,12 +190,11 @@ func (options *Options) validateOptions() error {
 		return errors.New("no engine specified")
 	}
 
-	options.dedupe()
+	if options.Delay < 0 {
+		return errors.New("delay can't be negative")
+	} else {
+		options.delay = time.Duration(options.Delay) * time.Second
+	}
 
 	return nil
 }
-
-func (options *Options) dedupe() {
-	options.Engine = sliceutil.Dedupe(options.Engine)
-	options.Query = sliceutil.Dedupe(options.Query)
-}

runner/runner.go

@@ -17,6 +17,7 @@ import (
 	"github.com/projectdiscovery/uncover/uncover/agent/censys"
 	"github.com/projectdiscovery/uncover/uncover/agent/fofa"
 	"github.com/projectdiscovery/uncover/uncover/agent/shodan"
+	"go.uber.org/ratelimit"
 )
 
 func init() {
@@ -42,6 +43,18 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 	if !r.options.Provider.HasKeys() {
 		return errors.New("no keys provided")
 	}
+
+	var censysRateLimiter, fofaRateLimiter, shodanRateLimiter ratelimit.Limiter
+	if r.options.Delay > 0 {
+		censysRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
+		fofaRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
+		shodanRateLimiter = ratelimit.New(1, ratelimit.Per(r.options.delay))
+	} else {
+		censysRateLimiter = ratelimit.NewUnlimited()
+		fofaRateLimiter = ratelimit.NewUnlimited()
+		shodanRateLimiter = ratelimit.NewUnlimited()
+	}
+
 	var agents []uncover.Agent
 	// declare clients
 	for _, engine := range r.options.Engine {
@@ -51,11 +64,11 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		)
 		switch engine {
 		case "shodan":
-			agent, err = shodan.New()
+			agent, err = shodan.NewWithOptions(&uncover.AgentOptions{RateLimiter: shodanRateLimiter})
 		case "censys":
-			agent, err = censys.New()
+			agent, err = censys.NewWithOptions(&uncover.AgentOptions{RateLimiter: censysRateLimiter})
 		case "fofa":
-			agent, err = fofa.New()
+			agent, err = fofa.NewWithOptions(&uncover.AgentOptions{RateLimiter: fofaRateLimiter})
 		default:
 			err = errors.New("unknown agent type")
 		}

uncover/agent.go

@@ -1,5 +1,11 @@
 package uncover
 
+import "go.uber.org/ratelimit"
+
+type AgentOptions struct {
+	RateLimiter ratelimit.Limiter
+}
+
 type Query struct {
 	Query string
 	Limit int

uncover/agent/censys/censys.go

@@ -16,12 +16,18 @@ const (
 	MaxPerPage = 100
 )
 
-type Agent struct{}
+type Agent struct {
+	options *uncover.AgentOptions
+}
 
 func New() (uncover.Agent, error) {
 	return &Agent{}, nil
 }
 
+func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) {
+	return &Agent{options: options}, nil
+}
+
 func (agent *Agent) Name() string {
 	return "censys"
 }
@@ -69,6 +75,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, censysRequest
 	}
 	request.Header.Set("Accept", "application/json")
 	request.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret)
+	agent.options.RateLimiter.Take()
 	return session.Do(request)
 }
 

uncover/agent/fofa/fofa.go

@@ -18,12 +18,18 @@ const (
 	Size   = 100
 )
 
-type Agent struct{}
+type Agent struct {
+	options *uncover.AgentOptions
+}
 
 func New() (uncover.Agent, error) {
 	return &Agent{}, nil
 }
 
+func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) {
+	return &Agent{options: options}, nil
+}
+
 func (agent *Agent) Name() string {
 	return "fofa"
 }
@@ -71,6 +77,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, fofaRequest *
 		return nil, err
 	}
 	request.Header.Set("Accept", "application/json")
+	agent.options.RateLimiter.Take()
 	return session.Do(request)
 }
 

uncover/agent/shodan/shodan.go

@@ -15,12 +15,18 @@ const (
 	URL = "https://api.shodan.io/shodan/host/search?key=%s&query=%s&page=%d"
 )
 
-type Agent struct{}
+type Agent struct {
+	options *uncover.AgentOptions
+}
 
 func New() (uncover.Agent, error) {
 	return &Agent{}, nil
 }
 
+func NewWithOptions(options *uncover.AgentOptions) (uncover.Agent, error) {
+	return &Agent{options: options}, nil
+}
+
 func (agent *Agent) Name() string {
 	return "shodan"
 }
@@ -68,6 +74,7 @@ func (agent *Agent) queryURL(session *uncover.Session, URL string, shodanRequest
 	if err != nil {
 		return nil, err
 	}
+	agent.options.RateLimiter.Take()
 	return session.Do(request)
 }