Merge remote-tracking branch 'origin'

Author: ehsandeep

.github/workflows/build-test.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/lint-test.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with: 
-          go-version: 1.20.x
+          go-version: 1.21.x
       - name: Run golangci-lint
         uses: golangci/[email protected]
         with:

.github/workflows/release-binary.yml

@@ -18,7 +18,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v4
         with: 
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: "Create release on GitHub"
         uses: goreleaser/goreleaser-action@v3

.github/workflows/release-test.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: release test
         uses: goreleaser/goreleaser-action@v4

README.md

@@ -89,8 +89,8 @@ OUTPUT:
   -oI, -ip                 include host IP in output (-active only)
 
 CONFIGURATION:
-  -config string                flag config file (default "$HOME/.config/subfinder/config.yaml")
-  -pc, -provider-config string  provider config file (default "$HOME/.config/subfinder/provider-config.yaml")
+  -config string                flag config file (default "$CONFIG/subfinder/config.yaml")
+  -pc, -provider-config string  provider config file (default "$CONFIG/subfinder/provider-config.yaml")
   -r string[]                   comma separated list of resolvers to use
   -rL, -rlist string            file containing list of resolvers to use
   -nW, -active                  display active subdomains only
@@ -117,140 +117,15 @@ OPTIMIZATION:
 go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
 ```
 
-## Post Installation Instructions
-
-`subfinder` can be used right after the installation, however the following services require configuring API keys to work:
-
-[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com), [BuiltWith](https://api.builtwith.com/domain-api)
-
-You can also use the `subfinder -ls` command to display all the available sources.
-
-These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys
-can be specified for each of these services from which one of them will be used for enumeration.
-
-Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix` and `360quake`, need to be separated with a colon (`:`).
-
-An example provider config file:
-
-```yaml
-binaryedge:
-  - 0bf8919b-aab9-42e4-9574-d3b639324597
-  - ac244e2f-b635-4581-878a-33f4e79a2c13
-censys:
-  - ac244e2f-b635-4581-878a-33f4e79a2c13:dd510d6e-1b6e-4655-83f6-f347b363def9
-certspotter: []
-passivetotal:
-  - [email protected]:sample_password
-redhuntlabs:
-  - ENDPOINT:API_TOKEN
-  - https://reconapi.redhuntlabs.com/community/v1/domains/subdomains:joEPzJJp2AuOCw7teAj63HYrPGnsxuPQ
-securitytrails: []
-shodan:
-  - AAAAClP1bJJSRMEYJazgwhJKrggRwKA
-github:
-  - ghp_lkyJGU3jv1xmwk4SDXavrLDJ4dl2pSJMzj4X
-  - ghp_gkUuhkIYdQPj13ifH4KA3cXRn8JD2lqir2d4
-zoomeyeapi:
-  - 4f73021d-ff95-4f53-937f-83d6db719eec
-quake:
-  - 0cb9030c-0a40-48a3-b8c4-fca28e466ba3
-facebook:
-  - APP_ID:APP_SECRET
-intelx:
-  - HOST:API_KEY
-  - 2.intelx.io:s4324-b98b-41b2-220e8-3320f6a1284d
-```
-
-Note: RedHunt Labs's [Attack Surface Recon API](https://devportal.redhuntlabs.com/) has different API endpoints depending on the user's subscription. Make sure to add the appropriate endpoint before running any scans.
-
-# Running Subfinder
-
-To run the tool on a target, just use the following command.
-
-```console
-subfinder -d hackerone.com
-
-               __    _____           __         
-   _______  __/ /_  / __(_)___  ____/ /__  _____
-  / ___/ / / / __ \/ /_/ / __ \/ __  / _ \/ ___/
- (__  ) /_/ / /_/ / __/ / / / / /_/ /  __/ /    
-/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9
-
-		projectdiscovery.io
-
-Use with caution. You are responsible for your actions
-Developers assume no liability and are not responsible for any misuse or damage.
-By using subfinder, you also agree to the terms of the APIs used.
-
-[INF] Enumerating subdomains for hackerone.com
-
-www.hackerone.com
-support.hackerone.com
-links.hackerone.com
-api.hackerone.com
-o1.email.hackerone.com
-go.hackerone.com
-3d.hackerone.com
-resources.hackerone.com
-a.ns.hackerone.com
-b.ns.hackerone.com
-mta-sts.hackerone.com
-docs.hackerone.com
-mta-sts.forwarding.hackerone.com
-gslink.hackerone.com
-hackerone.com
-info.hackerone.com
-mta-sts.managed.hackerone.com
-events.hackerone.com
-
-[INF] Found 18 subdomains for hackerone.com in 3 seconds 672 milliseconds
-```
-
-The subdomains discovered can be piped to other tools too. For example, you can pipe the discovered subdomains to [`httpx`](https://github.com/projectdiscovery/httpx) which will then find
-running HTTP servers on the host.
-
-```console
-echo hackerone.com | subfinder -silent | httpx -silent
-
-http://hackerone.com
-http://www.hackerone.com
-http://docs.hackerone.com
-http://api.hackerone.com
-https://docs.hackerone.com
-http://mta-sts.managed.hackerone.com
-```
+Learn about more ways to install subfinder here: https://docs.projectdiscovery.io/tools/subfinder/install.
 
-<table>
-<tr>
-<td>  
-
-## Subfinder with docker
-
-Pull the latest tagged [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder) docker image:
-
-```sh
-docker pull projectdiscovery/subfinder:latest
-```
-
-Running `subfinder` using the docker image:
-
-```sh
-docker run projectdiscovery/subfinder:latest -d hackerone.com
-```
-
-Running `subfinder` using the docker image, with a local config file:
+## Post Installation Instructions
 
-```sh
-docker run -v $HOME/.config/subfinder:/root/.config/subfinder -t projectdiscovery/subfinder -d hackerone.com
-```
+`subfinder` can be used right after the installation, however many sources required API keys to work. Learn more here: https://docs.projectdiscovery.io/tools/subfinder/install#post-install-configuration.
 
-</td>
-</tr>
-</table>
+## Running Subfinder
 
-<table>
-<tr>
-<td>  
+Learn about how to run Subfinder here: https://docs.projectdiscovery.io/tools/subfinder/running.
 
 ## Subfinder Go library
 

v2/go.mod

@@ -1,19 +1,19 @@
 module github.com/projectdiscovery/subfinder/v2
 
-go 1.20
+go 1.21
 
 require (
 	github.com/corpix/uarand v0.2.0
 	github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd
 	github.com/json-iterator/go v1.1.12
 	github.com/lib/pq v1.10.9
 	github.com/projectdiscovery/chaos-client v0.5.1
-	github.com/projectdiscovery/dnsx v1.1.4
+	github.com/projectdiscovery/dnsx v1.1.6
 	github.com/projectdiscovery/fdmax v0.0.4
-	github.com/projectdiscovery/gologger v1.1.11
-	github.com/projectdiscovery/ratelimit v0.0.9
-	github.com/projectdiscovery/retryablehttp-go v1.0.26
-	github.com/projectdiscovery/utils v0.0.54
+	github.com/projectdiscovery/gologger v1.1.12
+	github.com/projectdiscovery/ratelimit v0.0.23
+	github.com/projectdiscovery/retryablehttp-go v1.0.42
+	github.com/projectdiscovery/utils v0.0.72
 	github.com/rs/xid v1.5.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
@@ -28,16 +28,17 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/akrylysov/pogreb v0.10.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/charmbracelet/glamour v0.6.0 // indirect
 	github.com/cheggaaa/pb/v3 v3.1.4 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
-	github.com/dsnet/compress v0.0.1 // indirect
+	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/gaukas/godicttls v0.0.4 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -46,35 +47,36 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/mholt/archiver v3.1.1+incompatible // indirect
+	github.com/mholt/archiver/v3 v3.5.1 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.25 // indirect
 	github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.15.1 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
-	github.com/projectdiscovery/cdncheck v1.0.1 // indirect
-	github.com/projectdiscovery/fastdialer v0.0.37 // indirect
-	github.com/projectdiscovery/hmap v0.0.18 // indirect
+	github.com/projectdiscovery/cdncheck v1.0.9 // indirect
+	github.com/projectdiscovery/fastdialer v0.0.51 // indirect
+	github.com/projectdiscovery/hmap v0.0.33 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
-	github.com/quic-go/quic-go v0.37.4 // indirect
-	github.com/refraction-networking/utls v1.5.2 // indirect
+	github.com/quic-go/quic-go v0.37.7 // indirect
+	github.com/refraction-networking/utls v1.5.4 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
-	github.com/tidwall/btree v1.4.3 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/buntdb v1.3.0 // indirect
-	github.com/tidwall/gjson v1.14.3 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
@@ -88,11 +90,11 @@ require (
 	github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.11.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
@@ -102,13 +104,13 @@ require (
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
-	github.com/miekg/dns v1.1.55 // indirect
+	github.com/miekg/dns v1.1.56 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/projectdiscovery/goflags v0.1.20
-	github.com/projectdiscovery/retryabledns v1.0.35 // indirect
+	github.com/projectdiscovery/goflags v0.1.34
+	github.com/projectdiscovery/retryabledns v1.0.50 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 )