diff --git a/README.md b/README.md index db46be5b..aa25848e 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,12 @@ This will display help for the tool. Here are all the switches it supports. ```console +Usage: + ./httpx [flags] + +Flags: +httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library. + Usage: ./httpx [flags] @@ -124,7 +130,8 @@ HEADLESS: -ho, -headless-options string[] start headless chrome with additional options -esb, -exclude-screenshot-bytes enable excluding screenshot bytes from json output -ehb, -exclude-headless-body enable excluding headless header from json output - -st, -screenshot-timeout int set timeout for screenshot in seconds (default 10) + -st, -screenshot-timeout value set timeout for screenshot in seconds (default 10s) + -sid, -screenshot-idle value set idle time before taking screenshot in seconds (default 1s) MATCHERS: -mc, -match-code string match response with specified status code (-mc 200,302) @@ -134,24 +141,25 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string[] match response with specified string (-ms admin) -mr, -match-regex string[] match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (google, cloudfront, fastly) + -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition EXTRACTOR: -er, -extract-regex string[] display response content with matched regex - -ep, -extract-preset string[] display response content matched by a pre-defined regex (ipv4,mail,url) + -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) -fep, -filter-error-page filter response with ML based error page detection + -fd, -filter-duplicates filter out near-duplicate responses (only first response is retained) -fl, -filter-length string filter response with specified content length (-fl 23,33) -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000) -fs, -filter-string string[] filter response with specified string (-fs admin) -fe, -filter-regex string[] filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (google, cloudfront, fastly) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition -strip strips all tags in response. supported formats: html,xml (default html) @@ -234,6 +242,7 @@ DEBUG: -v, -verbose verbose mode -si, -stats-interval int number of seconds to wait between showing a statistics update (default: 5) -nc, -no-color disable colors in cli output + -tr, -trace trace OPTIMIZATIONS: -nf, -no-fallback display both probed protocol (HTTPS and HTTP) @@ -248,7 +257,9 @@ OPTIMIZATIONS: CLOUD: -auth configure projectdiscovery cloud (pdcp) api key (default true) + -ac, -auth-config string configure projectdiscovery cloud (pdcp) api key credential file -pd, -dashboard upload / view output in projectdiscovery cloud (pdcp) UI dashboard + -tid, -team-id string upload asset results to given team id (optional) -aid, -asset-id string upload new assets to existing asset id (optional) -aname, -asset-name string assets group name to set (optional) -pdu, -dashboard-upload string upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 3c890e29..4f4aa9a7 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -138,5 +138,8 @@ func setupOptionalAssetUpload(opts *runner.Options) *pdcp.UploadWriter { // silently ignore writer.SetAssetGroupName(opts.AssetName) } + if opts.TeamID != "" { + writer.SetTeamID(opts.TeamID) + } return writer } diff --git a/common/errorpageclassifier/clf.gob b/common/errorpageclassifier/clf.gob deleted file mode 100644 index f6978625..00000000 Binary files a/common/errorpageclassifier/clf.gob and /dev/null differ diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 590bc046..d5027b79 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -76,7 +76,7 @@ func New(options *Options) (*HTTPX, error) { var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying retryablehttpOptions.Timeout = httpx.Options.Timeout retryablehttpOptions.RetryMax = httpx.Options.RetryMax - + retryablehttpOptions.Trace = options.Trace handleHSTS := func(req *http.Request) { if req.Response.Header.Get("Strict-Transport-Security") == "" { return diff --git a/common/httpx/option.go b/common/httpx/option.go index 3f5d2f7f..6ef61919 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -49,6 +49,7 @@ type Options struct { NetworkPolicy *networkpolicy.NetworkPolicy CDNCheckClient *cdncheck.Client Protocol Proto + Trace bool } // DefaultOptions contains the default options diff --git a/common/pagetypeclassifier/clf.gob b/common/pagetypeclassifier/clf.gob new file mode 100644 index 00000000..7624ee77 Binary files /dev/null and b/common/pagetypeclassifier/clf.gob differ diff --git a/common/errorpageclassifier/dataset.txt b/common/pagetypeclassifier/dataset.txt similarity index 63% rename from common/errorpageclassifier/dataset.txt rename to common/pagetypeclassifier/dataset.txt index 1f9ee4e3..c55ab2ef 100644 --- a/common/errorpageclassifier/dataset.txt +++ b/common/pagetypeclassifier/dataset.txt @@ -108,7 +108,7 @@ Now viewing: Image Gallery. Enjoy a visual tour of our activities.||nonerror You're on our FAQ page. Get answers to common questions.||nonerror Welcome to the Blog section. Engage with our thoughts and insights.||nonerror This is the Discussion Forum. Join in, ask questions, or help others.||nonerror -You're on the Login page. Enter your credentials to access your account.||nonerror +You're on the Login page. Enter your credentials to access your account.||login Welcome to the Sign-Up page. Join our community today.||nonerror This is your User Dashboard. Manage your account and settings here.||nonerror You've reached the Checkout page. Review your order and proceed to payment.||nonerror @@ -198,4 +198,183 @@ You've successfully added the item to your cart!||nonerror Success! Your password has been updated||nonerror Welcome back! You have successfully logged in||nonerror Great job! Your profile has been updated||nonerror -Your message was sent successfully. We'll get back to you shortly||nonerror \ No newline at end of file +Your message was sent successfully. We'll get back to you shortly||nonerror +Welcome to the Login page. Please sign in to continue.||login +Please enter your username and password on the login page.||login +You have reached the login page. Access your account by logging in.||login +Login required. Please authenticate to access this page.||login +Welcome back! Please log in to your account.||login +Sign in to your account on this login page.||login +Secure Login: Enter your credentials to proceed.||login +This is the login page. Please enter your email and password.||login +Access denied. Please log in to continue.||login +You're on the login page. Forgot your password? Click here to reset.||login +User Login: Please provide your username and password.||login +Login to your account to access exclusive features.||login +Authentication required. Please log in.||login +Welcome back! Sign in to access your dashboard.||login +Please log in to proceed to the checkout page.||login +Member login: Enter your credentials below.||login +Staff login portal. Please enter your login details.||login +Customer login: Sign in to view your orders.||login +Partner login: Please authenticate to access partner resources.||login +Administrator login page. Enter your admin credentials.||login +Please log in to access your profile settings.||login +Login successful. Redirecting to your account dashboard.||login +Incorrect password. Please try again.||login +Session expired. Please log in again.||login +Welcome to the secure login page. Your privacy is important to us.||login +Access restricted. Please log in to view this content.||login +Please log in to access the members-only area.||login +Sign in with your social media account on the login page.||login +New user? Register here or log in if you already have an account.||login +Log in to participate in the forum discussions.||login +Access your account by logging in here.||login +Please log in to access your personalized dashboard.||login +Enter your login details to continue.||login +Login Page: Securely enter your credentials.||login +Welcome to the user login portal.||login +Sign in to manage your account settings.||login +This is the login screen. Please authenticate.||login +Returning user? Please log in.||login +Please log in to view your messages.||login +Log in to access premium content.||login +Authentication page: Enter your username and password.||login +Please enter your login information to proceed.||login +User authentication required. Please log in.||login +Log in now to unlock exclusive features.||login +Sign in to check your account balance.||login +Welcome back! Please enter your login credentials.||login +Member login area: Access restricted content by logging in.||login +Please sign in to continue to your profile.||login +Staff members, please log in to access internal resources.||login +Enter your email and password to log in.||login +Login required to view this page. Please sign in.||login +Access your profile by logging into your account.||login +Please provide your login credentials to access the system.||login +Log in to track your order status.||login +Welcome to the employee login page.||login +Secure area: Please log in to continue.||login +Please log in to update your preferences.||login +Sign in to access your learning materials.||login +Please authenticate to proceed to the next step.||login +Login Page: Your session has expired, please log in again.||login +Welcome back! Enter your credentials to sign in.||login +Client login: Access your project details here.||login +Agent login portal: Please sign in with your ID.||login +Enter your user ID and password to log in.||login +Log in to view your subscription details.||login +Login Page: Forgot your password? Click here to reset it.||login +Access restricted to authorized users only. Please log in.||login +Vendor login: Manage your listings by logging in.||login +Please log in to access your saved items.||login +Log in to participate in our online courses.||login +Sign in to view your appointment schedule.||login +Welcome to the admin login page.||login +Please enter your credentials to log in securely.||login +Log in to view your recent activities.||login +Authentication needed. Please sign in to proceed.||login +Member login: Keep me signed in checkbox available.||login +Log in with your email or username.||login +Access your account dashboard by logging in.||login +Sign in to post comments on articles.||login +Please log in to access your billing information.||login +Log in to access your personalized recommendations.||login +Please sign in to view your shopping cart.||login +Enter your credentials to log in and start shopping.||login +Welcome to the customer login page. Sign in to continue.||login +Authentication required. Please log in with your secure ID.||login +Log in to access exclusive member discounts.||login +Please log in to view and manage your wishlist.||login +Sign in to access your event tickets and details.||login +Faculty login: Please enter your staff ID and password.||login +Log in to access your investment portfolio.||login +Access your medical records by logging in securely.||login +Please sign in to continue to the payment gateway.||login +Login required to access your order history.||login +Welcome back! Log in to resume your session.||login +Please log in to submit your application.||login +Enter your username and password to log in to the portal.||login +Student login: Access your course materials by signing in.||login +Log in to customize your news feed preferences.||login +Please authenticate to access your secure messages.||login +Sign in to sync your data across devices.||login +Log in to join the live webinar.||login +Please log in to access your reservation details.||login +Welcome to the supplier login page.||login +Log in to access your support tickets.||login +Enter your credentials to log in and view analytics.||login +Please sign in to access developer resources.||login +Login required to view confidential documents.||login +Log in to participate in the survey.||login +Please authenticate to access the admin dashboard.||login +Sign in to view your loyalty points balance.||login +Log in to manage your email subscriptions.||login +Please log in to proceed with the enrollment process.||login +Access your download history by logging in.||login +Welcome back! Please log in to renew your membership.||login +Enter your employee ID to log in to the time tracking system.||login +Log in to update your security settings.||login +Please sign in to access your saved searches.||login +Authentication required for accessing project files.||login +Log in to collaborate with your team members.||login +Please enter your PIN and password to log in.||login +Sign in to access your fitness progress dashboard.||login +Log in to check your test results.||login +Please log in to schedule your appointments.||login +Welcome to the volunteer login page.||login +Log in to view your donation history.||login +Please authenticate to access the control panel.||login +Sign in to review and accept your job offer.||login +Log in to access premium tutorials and guides.||login +Please log in to manage your API keys.||login +Please log in with your email address and password.||login +Enter your username and password to access your account.||login +Sign in to your account using your email and password.||login +Welcome back! Please enter your login credentials.||login +Email address: [input field] Password: [input field]||login +Login to your account. Don't have one? Sign up now.||login +Username: [input field] Password: [input field] Remember me?||login +Forgot your password? Click here to reset it.||login +Please enter your email and password to continue.||login +Secure login portal. Enter credentials below.||login +Access your account by logging in below.||login +Remember me on this device.||login +Login required. Please sign in to proceed.||login +Forgot password? Reset it here.||login +Sign in with your email address and password.||login +Welcome! Please log in to your account.||login +User login: Enter your email and password.||login +Email: [input field] Password: [input field]||login +Please authenticate by entering your login details.||login +Sign in to your account or register for a new one.||login +Login page: Access restricted to authorized users only.||login +Need help logging in? Click here.||login +Enter your credentials to log in.||login +Keep me signed in.||login +Please sign in to access exclusive content.||login +Welcome back! Sign in to your dashboard.||login +Forgot your username or password? Retrieve them here.||login +Log in using your email or username.||login +Authentication required. Please log in.||login +Password recovery: Reset your password now.||login +Log in to manage your account settings.||login +Sign in to continue to checkout.||login +Enter your login information below.||login +Sign in to access your personalized dashboard.||login +Welcome to the member login page.||login +Already have an account? Log in here.||login +Enter email and password to sign in.||login +Sign in to your profile.||login +Member login: Access your account here.||login +Please log in to continue.||login +Enter your password to log in.||login +Sign in to view your messages.||login +Login to your profile to see updates.||login +Log in to your account to access features.||login +Please provide your username and password.||login +Log in to manage your subscriptions.||login +Sign in using your credentials.||login +Access denied. Please log in first.||login +Authentication portal. Enter login details.||login +Need an account? Sign up or log in if you already have one.||login diff --git a/common/errorpageclassifier/errorpageclassifier.go b/common/pagetypeclassifier/pagetypeclassifier.go similarity index 73% rename from common/errorpageclassifier/errorpageclassifier.go rename to common/pagetypeclassifier/pagetypeclassifier.go index d916d7ce..67eb3e09 100644 --- a/common/errorpageclassifier/errorpageclassifier.go +++ b/common/pagetypeclassifier/pagetypeclassifier.go @@ -1,4 +1,4 @@ -package errorpageclassifier +package pagetypeclassifier import ( _ "embed" @@ -10,19 +10,19 @@ import ( //go:embed clf.gob var classifierData []byte -type ErrorPageClassifier struct { +type PageTypeClassifier struct { classifier *naive_bayes.NaiveBayesClassifier } -func New() *ErrorPageClassifier { +func New() *PageTypeClassifier { classifier, err := naive_bayes.NewClassifierFromFileData(classifierData) if err != nil { panic(err) } - return &ErrorPageClassifier{classifier: classifier} + return &PageTypeClassifier{classifier: classifier} } -func (n *ErrorPageClassifier) Classify(html string) string { +func (n *PageTypeClassifier) Classify(html string) string { text := htmlToText(html) if text == "" { return "other" diff --git a/common/errorpageclassifier/errorpageclassifier_test.go b/common/pagetypeclassifier/pagetypeclassifier_test.go similarity index 87% rename from common/errorpageclassifier/errorpageclassifier_test.go rename to common/pagetypeclassifier/pagetypeclassifier_test.go index 35923b28..30571f0b 100644 --- a/common/errorpageclassifier/errorpageclassifier_test.go +++ b/common/pagetypeclassifier/pagetypeclassifier_test.go @@ -1,4 +1,4 @@ -package errorpageclassifier +package pagetypeclassifier import ( "testing" @@ -6,8 +6,9 @@ import ( "github.com/stretchr/testify/assert" ) -func TestErrorPageClassifier(t *testing.T) { - t.Run("test creation of new ErrorPageClassifier", func(t *testing.T) { +func TestPageTypeClassifier(t *testing.T) { + + t.Run("test creation of new PageTypeClassifier", func(t *testing.T) { epc := New() assert.NotNil(t, epc) }) diff --git a/go.mod b/go.mod index 7dad8b49..2827bb53 100644 --- a/go.mod +++ b/go.mod @@ -16,29 +16,28 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 - github.com/microcosm-cc/bluemonday v1.0.26 + github.com/microcosm-cc/bluemonday v1.0.27 github.com/miekg/dns v1.1.59 // indirect - github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 github.com/projectdiscovery/cdncheck v1.1.0 - github.com/projectdiscovery/clistats v0.1.0 - github.com/projectdiscovery/dsl v0.1.10 - github.com/projectdiscovery/fastdialer v0.2.3 + github.com/projectdiscovery/clistats v0.1.1 + github.com/projectdiscovery/dsl v0.2.5 + github.com/projectdiscovery/fastdialer v0.2.9 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.62 - github.com/projectdiscovery/gologger v1.1.19 - github.com/projectdiscovery/hmap v0.0.54 + github.com/projectdiscovery/goflags v0.1.64 + github.com/projectdiscovery/gologger v1.1.25 + github.com/projectdiscovery/hmap v0.0.61 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/ratelimit v0.0.49 - github.com/projectdiscovery/rawhttp v0.1.61 - github.com/projectdiscovery/retryablehttp-go v1.0.72 - github.com/projectdiscovery/tlsx v1.1.6 - github.com/projectdiscovery/useragent v0.0.65 - github.com/projectdiscovery/utils v0.2.4 - github.com/projectdiscovery/wappalyzergo v0.1.12 + github.com/projectdiscovery/ratelimit v0.0.54 + github.com/projectdiscovery/rawhttp v0.1.68 + github.com/projectdiscovery/retryablehttp-go v1.0.81 + github.com/projectdiscovery/tlsx v1.1.7 + github.com/projectdiscovery/useragent v0.0.69 + github.com/projectdiscovery/utils v0.2.12 + github.com/projectdiscovery/wappalyzergo v0.1.22 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.9.0 @@ -46,12 +45,15 @@ require ( go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.26.0 - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 + golang.org/x/net v0.29.0 + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 ) -require github.com/weppos/publicsuffix-go v0.30.2 +require ( + github.com/go-viper/mapstructure/v2 v2.1.0 + github.com/weppos/publicsuffix-go v0.30.2 +) require ( aead.dev/minisign v0.2.0 // indirect @@ -59,21 +61,23 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/alecthomas/chroma v0.10.0 // indirect + github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect - github.com/charmbracelet/glamour v0.6.0 // indirect + github.com/charmbracelet/glamour v0.8.0 // indirect + github.com/charmbracelet/lipgloss v0.13.0 // indirect + github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.8 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dlclark/regexp2 v1.11.0 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.16.0 // indirect @@ -96,13 +100,13 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.1 // indirect + github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect @@ -110,12 +114,12 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/freeport v0.0.5 // indirect + github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.70 // indirect + github.com/projectdiscovery/retryabledns v1.0.78 // indirect github.com/refraction-networking/utls v1.6.7 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/sashabaranov/go-openai v1.15.3 // indirect @@ -140,16 +144,16 @@ require ( github.com/ysmood/got v0.34.1 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect - github.com/yuin/goldmark v1.5.4 // indirect - github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/yuin/goldmark v1.7.4 // indirect + github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/term v0.24.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 1d9a8ac0..e9127d76 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,12 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= -github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= +github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= +github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= @@ -27,9 +31,10 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= @@ -37,8 +42,14 @@ github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= -github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= +github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= +github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= +github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= +github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= +github.com/charmbracelet/x/ansi v0.3.2 h1:wsEwgAN+C9U06l9dCVMX0/L3x7ptvY1qmjMwyfE6USY= +github.com/charmbracelet/x/ansi v0.3.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= @@ -55,9 +66,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= @@ -78,6 +88,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -110,7 +122,6 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -121,6 +132,8 @@ github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA= github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= @@ -161,23 +174,19 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= -github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= -github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -187,9 +196,8 @@ github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1 github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= -github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= -github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= +github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= @@ -222,56 +230,56 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= -github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs= -github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.1.10 h1:FBD5Muiwj3OdkDAYvmltQ0rcVMpLZK5IqNZi9Pd/gb0= -github.com/projectdiscovery/dsl v0.1.10/go.mod h1:Kg9snSkjTPZ2qiIDr4jjFI46FgJzvqz48qcHWUWY7lw= -github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0= -github.com/projectdiscovery/fastdialer v0.2.3/go.mod h1:a0BKvETrO1EAohUYp9gwtbbce0hKD1qGrTegyAUqyRo= +github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= +github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= +github.com/projectdiscovery/dsl v0.2.5 h1:SJzIPGryApvNuAMa/VCe2yPzIDbkoz/pyH8Zt3Wlk+0= +github.com/projectdiscovery/dsl v0.2.5/go.mod h1:aLyfr+br+cgwhHUWdQkIYJz+riWABaJ76jKcPYkcWYE= +github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA= +github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= -github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= +github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= +github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc= -github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk= -github.com/projectdiscovery/gologger v1.1.19 h1:b7cU32XuDrDiwhr7hlDeE6mfj/nENBtHEohe51txJCE= -github.com/projectdiscovery/gologger v1.1.19/go.mod h1:DbeKwx9IEfcvnclImX5gBlhIKUuOZwOM5itdpYXl+54= +github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= +github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= +github.com/projectdiscovery/gologger v1.1.25 h1:bHarLQ/sCfBErU6QCarE1fuGpINEkyHu8zhYUWE2oMA= +github.com/projectdiscovery/gologger v1.1.25/go.mod h1:AmWrlwr9Xs/dRALoN3wjuNzdSpvNu0nFDXI/4+mII7Q= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.54 h1:b3pdQZwCw4is3xiL2jBx7SJZcYaf/7vtozY7bjUzO/s= -github.com/projectdiscovery/hmap v0.0.54/go.mod h1:j0oakxYOWEfk29wRq5gQgrCv1JnfAfzGaMsRWwEas80= +github.com/projectdiscovery/hmap v0.0.61 h1:1lmb2MfDWY7e3Xq7WufcbecD6MKYMj/7yCmKp7nueDc= +github.com/projectdiscovery/hmap v0.0.61/go.mod h1:DQWTC4DYnSQvMgxjYv43pdQhH0gjyAAZMx28xYOCNTA= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ= -github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k= -github.com/projectdiscovery/rawhttp v0.1.61 h1:EbskCj6kkDSG31sO5zEUFTqHp9ltccG1DdcI+MCdahQ= -github.com/projectdiscovery/rawhttp v0.1.61/go.mod h1:5XmDAKph9pLVnh87zjL+vXDpfG5W8Gz5N2BAtYTkuVU= -github.com/projectdiscovery/retryabledns v1.0.70 h1:2yFMqQ4v3tgI9ORjlPH60h5QIs2EXxHVGCaxrOA1ZlI= -github.com/projectdiscovery/retryabledns v1.0.70/go.mod h1:Ld/RLVsG7d+wlNcye9xcuPTjGHLF9XO8w34GLRKsNis= -github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= -github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= +github.com/projectdiscovery/ratelimit v0.0.54 h1:VwrFLPSfWle5Hg7AvwQkDd4EWDzYWTzd2mDTXTrqvmA= +github.com/projectdiscovery/ratelimit v0.0.54/go.mod h1:yGIqMaT8vKG+4mlCqkWOWAvBtvWUlQeanZae31E09cY= +github.com/projectdiscovery/rawhttp v0.1.68 h1:6oXatkSkO9a8ybtbhgowQ5wXG0QxmH8a/Ulk4DRsLMM= +github.com/projectdiscovery/rawhttp v0.1.68/go.mod h1:gEF17M4VdXbU9Gn9sZXOY75/bXSduq6XuvZTGxw2/2A= +github.com/projectdiscovery/retryabledns v1.0.78 h1:CAp1T3r40u1GgWo0TLFgfzA5o0o+XSMs5RMiq7ieot4= +github.com/projectdiscovery/retryabledns v1.0.78/go.mod h1:DX75WD3/fmcnd5coOLQjPV9jsOokbkf5BZdmZm/mhyQ= +github.com/projectdiscovery/retryablehttp-go v1.0.81 h1:F/sJauqgYg4fgqEZHfLUZx0GoJcKNMcJkKHfQES/2As= +github.com/projectdiscovery/retryablehttp-go v1.0.81/go.mod h1:P3tO8Rg61T7kot7bqXdeJPsowEVRzWQ4IydqV596ZnI= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= -github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= -github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= -github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= -github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= -github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= -github.com/projectdiscovery/wappalyzergo v0.1.12 h1:usZmlyiNDRvdYstZpFQH11cx413Ppykhh5pg6zVvcY4= -github.com/projectdiscovery/wappalyzergo v0.1.12/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl3yynSM= +github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM= +github.com/projectdiscovery/useragent v0.0.69 h1:6TkVMKjOBDqecUBBkXlTlBUfAUM8Bgn/58E3ZasndBs= +github.com/projectdiscovery/useragent v0.0.69/go.mod h1:Ss+Kf6XEjSQqhm7LmbEtOxKvGnnnR88Fx+0PG2PXuBY= +github.com/projectdiscovery/utils v0.2.12 h1:X0X08Km47gxLoIgc6nprWj0G28dEfRmSBLjO3/DkK8Q= +github.com/projectdiscovery/utils v0.2.12/go.mod h1:OaYUfawin0idXUZjwLFFiDpfmpf90hDF/lAubb2qWs4= +github.com/projectdiscovery/wappalyzergo v0.1.22 h1:7QnpeZy/txsvVcQNlL1n4xIu8IXgMxEsda3w6k6bShY= +github.com/projectdiscovery/wappalyzergo v0.1.22/go.mod h1:wnvmbC10pQTOoCKnCTmWKP20rpEtqrMJZvzuTuleeyw= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -359,13 +367,12 @@ github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= -github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= -github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg= +github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4= +github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= @@ -398,8 +405,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -420,7 +427,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -429,8 +435,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= @@ -440,8 +446,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -462,7 +468,6 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -472,8 +477,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -484,8 +489,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -499,8 +504,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/pdcp/utils.go b/internal/pdcp/utils.go index 301ef2aa..3c399802 100644 --- a/internal/pdcp/utils.go +++ b/internal/pdcp/utils.go @@ -5,9 +5,17 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) -func getAssetsDashBoardURL(id string) string { +func getAssetsDashBoardURL(id, teamID string) string { ux, _ := urlutil.Parse(pdcpauth.DashBoardURL) ux.Path = "/assets/" + id + if ux.Params == nil { + ux.Params = urlutil.NewOrderedParams() + } + if teamID != "" { + ux.Params.Add("team_id", teamID) + } else { + ux.Params.Add("team_id", NoneTeamID) + } ux.Update() return ux.String() } diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index 5fd023ab..61327eeb 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -31,12 +31,11 @@ const ( MaxChunkSize = 4 * unitutils.Mega // 4 MB xidRe = `^[a-z0-9]{20}$` teamIDHeader = "X-Team-Id" + NoneTeamID = "none" ) var ( xidRegex = regexp.MustCompile(xidRe) - // teamID if given - teamID = env.GetEnvOrDefault("PDCP_TEAM_ID", "") // EnableeUpload if set to true enables the upload feature HideAutoSaveMsg = env.GetEnvOrDefault("DISABLE_CLOUD_UPLOAD_WRN", false) EnableCloudUpload = env.GetEnvOrDefault("ENABLE_CLOUD_UPLOAD", false) @@ -54,6 +53,7 @@ type UploadWriter struct { assetGroupName string counter atomic.Int32 closed atomic.Bool + TeamID string } // NewUploadWriterCallback creates a new upload writer callback @@ -63,9 +63,10 @@ func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredential return nil, fmt.Errorf("no credentials provided") } u := &UploadWriter{ - creds: creds, - done: make(chan struct{}, 1), - data: make(chan runner.Result, 8), // default buffer size + creds: creds, + done: make(chan struct{}, 1), + data: make(chan runner.Result, 8), // default buffer size + TeamID: "", } var err error tmp, err := urlutil.Parse(creds.Server) @@ -111,6 +112,11 @@ func (u *UploadWriter) SetAssetGroupName(name string) { u.assetGroupName = name } +// SetTeamID sets the team id for the upload writer +func (u *UploadWriter) SetTeamID(id string) { + u.TeamID = id +} + func (u *UploadWriter) autoCommit(ctx context.Context) { // wait for context to be done defer func() { @@ -120,7 +126,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context) { if u.assetGroupID == "" { gologger.Verbose().Msgf("UI dashboard setup skipped, no results found to upload") } else { - gologger.Info().Msgf("Found %v results, View found results in dashboard : %v", u.counter.Load(), getAssetsDashBoardURL(u.assetGroupID)) + gologger.Info().Msgf("Found %v results, View found results in dashboard : %v", u.counter.Load(), getAssetsDashBoardURL(u.assetGroupID, u.TeamID)) } }() // temporary buffer to store the results @@ -185,7 +191,7 @@ func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error { // if successful, reset the buffer buff.Reset() // log in verbose mode - gologger.Warning().Msgf("Uploaded results chunk, you can view assets at %v", getAssetsDashBoardURL(u.assetGroupID)) + gologger.Warning().Msgf("Uploaded results chunk, you can view assets at %v", getAssetsDashBoardURL(u.assetGroupID, u.TeamID)) return nil } @@ -244,8 +250,8 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { req.URL.Update() req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) - if teamID != "" { - req.Header.Set(teamIDHeader, teamID) + if u.TeamID != "" { + req.Header.Set(teamIDHeader, u.TeamID) } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", "application/json") diff --git a/runner/banner.go b/runner/banner.go index eeb04d18..d546076c 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current Version of httpx -const Version = `v1.6.8` +const Version = `v1.6.9` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/headless.go b/runner/headless.go index c298cb76..481fd1cb 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -99,7 +99,7 @@ func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*B return engine, nil } -func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, headers []string) ([]byte, string, error) { +func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle time.Duration, headers []string) ([]byte, string, error) { page, err := b.engine.Page(proto.TargetCreateTarget{}) if err != nil { return nil, "", err @@ -126,7 +126,7 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, headers if err := page.WaitLoad(); err != nil { return nil, "", err } - _ = page.WaitIdle(1 * time.Second) + _ = page.WaitIdle(idle) screenshot, err := page.Screenshot(true, &proto.PageCaptureScreenshot{}) if err != nil { diff --git a/runner/options.go b/runner/options.go index 1a0265b9..c09944d1 100644 --- a/runner/options.go +++ b/runner/options.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "os" + "path/filepath" "regexp" "strings" "time" @@ -25,7 +26,7 @@ import ( "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/networkpolicy" - "github.com/projectdiscovery/utils/auth/pdcp" + pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" sliceutil "github.com/projectdiscovery/utils/slice" @@ -41,7 +42,10 @@ const ( DefaultOutputDirectory = "output" ) -var PDCPApiKey = "" +var ( + PDCPApiKey = "" + TeamIDEnv = env.GetEnvOrDefault("PDCP_TEAM_ID", "") +) // OnResultCallback (hostResult) type OnResultCallback func(Result) @@ -100,7 +104,8 @@ type ScanOptions struct { DisableStdin bool NoScreenshotBytes bool NoHeadlessBody bool - ScreenshotTimeout int + ScreenshotTimeout time.Duration + ScreenshotIdle time.Duration } func (s *ScanOptions) Clone() *ScanOptions { @@ -154,6 +159,7 @@ func (s *ScanOptions) Clone() *ScanOptions { NoScreenshotBytes: s.NoScreenshotBytes, NoHeadlessBody: s.NoHeadlessBody, ScreenshotTimeout: s.ScreenshotTimeout, + ScreenshotIdle: s.ScreenshotIdle, } } @@ -181,6 +187,7 @@ type Options struct { OutputMatchContentLength string OutputFilterStatusCode string OutputFilterErrorPage bool + FilterOutDuplicates bool OutputFilterContentLength string InputRawRequest string rawRequest string @@ -209,6 +216,7 @@ type Options struct { CSVOutput bool CSVOutputEncoding string PdcpAuth string + PdcpAuthCredFile string Silent bool Version bool Verbose bool @@ -304,7 +312,8 @@ type Options struct { HttpApiEndpoint string NoScreenshotBytes bool NoHeadlessBody bool - ScreenshotTimeout int + ScreenshotTimeout time.Duration + ScreenshotIdle time.Duration // HeadlessOptionalArguments specifies optional arguments to pass to Chrome HeadlessOptionalArguments goflags.StringSlice Protocol string @@ -318,10 +327,13 @@ type Options struct { AssetID string // AssetFileUpload AssetFileUpload string + TeamID string // OnClose adds a callback function that is invoked when httpx is closed // to be exact at end of existing closures OnClose func() + Trace bool + // Optional pre-created objects to reduce allocations Wappalyzer *wappalyzer.Wappalyze Networkpolicy *networkpolicy.NetworkPolicy @@ -373,7 +385,8 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.NoScreenshotBytes, "exclude-screenshot-bytes", "esb", false, "enable excluding screenshot bytes from json output"), flagSet.BoolVarP(&options.NoHeadlessBody, "exclude-headless-body", "ehb", false, "enable excluding headless header from json output"), - flagSet.IntVarP(&options.ScreenshotTimeout, "screenshot-timeout", "st", 10, "set timeout for screenshot in seconds"), + flagSet.DurationVarP(&options.ScreenshotTimeout, "screenshot-timeout", "st", 10*time.Second, "set timeout for screenshot in seconds"), + flagSet.DurationVarP(&options.ScreenshotIdle, "screenshot-idle", "sid", 1*time.Second, "set idle time before taking screenshot in seconds"), ) flagSet.CreateGroup("matchers", "Matchers", @@ -397,6 +410,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("filters", "Filters", flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"), flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"), + flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses (only first response is retained)"), flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"), @@ -492,6 +506,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"), + flagSet.BoolVarP(&options.Trace, "trace", "tr", false, "trace"), ) flagSet.CreateGroup("Optimizations", "Optimizations", @@ -508,7 +523,9 @@ func ParseOptions() *Options { flagSet.CreateGroup("cloud", "Cloud", flagSet.DynamicVar(&options.PdcpAuth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), + flagSet.StringVarP(&options.PdcpAuthCredFile, "auth-config", "ac", "", "configure projectdiscovery cloud (pdcp) api key credential file"), flagSet.BoolVarP(&options.AssetUpload, "dashboard", "pd", false, "upload / view output in projectdiscovery cloud (pdcp) UI dashboard"), + flagSet.StringVarP(&options.TeamID, "team-id", "tid", TeamIDEnv, "upload asset results to given team id (optional)"), flagSet.StringVarP(&options.AssetID, "asset-id", "aid", "", "upload new assets to existing asset id (optional)"), flagSet.StringVarP(&options.AssetName, "asset-name", "aname", "", "assets group name to set (optional)"), flagSet.StringVarP(&options.AssetFileUpload, "dashboard-upload", "pdu", "", "upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"), @@ -535,14 +552,19 @@ func ParseOptions() *Options { } } + if options.PdcpAuthCredFile != "" { + pdcpauth.PDCPCredFile = options.PdcpAuthCredFile + pdcpauth.PDCPDir = filepath.Dir(pdcpauth.PDCPCredFile) + } + // api key hierarchy: cli flag > env var > .pdcp/credential file if options.PdcpAuth == "true" { AuthWithPDCP() } else if len(options.PdcpAuth) == 36 { PDCPApiKey = options.PdcpAuth - ph := pdcp.PDCPCredHandler{} - if _, err := ph.GetCreds(); err == pdcp.ErrNoCreds { - apiServer := env.GetEnvOrDefault("PDCP_API_SERVER", pdcp.DefaultApiServer) + ph := pdcpauth.PDCPCredHandler{} + if _, err := ph.GetCreds(); err == pdcpauth.ErrNoCreds { + apiServer := env.GetEnvOrDefault("PDCP_API_SERVER", pdcpauth.DefaultApiServer) if validatedCreds, err := ph.ValidateAPIKey(PDCPApiKey, apiServer, "httpx"); err == nil { _ = ph.SaveCreds(validatedCreds) } @@ -625,7 +647,7 @@ func (options *Options) ValidateOptions() error { msg += fmt.Sprintf("%s flag is", last) } msg += " incompatible with silent flag" - return fmt.Errorf(msg) + return errors.New(msg) } } diff --git a/runner/runner.go b/runner/runner.go index 6a3511a0..4b806150 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -29,11 +29,12 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/corona10/goimagehash" + "github.com/mfonda/simhash" asnmap "github.com/projectdiscovery/asnmap/libs" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" - "github.com/projectdiscovery/httpx/common/errorpageclassifier" "github.com/projectdiscovery/httpx/common/hashes/jarm" + "github.com/projectdiscovery/httpx/common/pagetypeclassifier" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/networkpolicy" @@ -65,6 +66,7 @@ import ( "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/rawhttp" + converstionutil "github.com/projectdiscovery/utils/conversion" fileutil "github.com/projectdiscovery/utils/file" pdhttputil "github.com/projectdiscovery/utils/http" iputil "github.com/projectdiscovery/utils/ip" @@ -74,19 +76,20 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - hp *httpx.HTTPX - wappalyzer *wappalyzer.Wappalyze - scanopts ScanOptions - hm *hybrid.HybridMap - excludeCdn bool - stats clistats.StatisticsClient - ratelimiter ratelimit.Limiter - HostErrorsCache gcache.Cache[string, int] - browser *Browser - errorPageClassifier *errorpageclassifier.ErrorPageClassifier - pHashClusters []pHashCluster - httpApiEndpoint *Server + options *Options + hp *httpx.HTTPX + wappalyzer *wappalyzer.Wappalyze + scanopts ScanOptions + hm *hybrid.HybridMap + excludeCdn bool + stats clistats.StatisticsClient + ratelimiter ratelimit.Limiter + HostErrorsCache gcache.Cache[string, int] + browser *Browser + pageTypeClassifier *pagetypeclassifier.PageTypeClassifier // Include this for general page classification + pHashClusters []pHashCluster + simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection + httpApiEndpoint *Server } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -126,6 +129,7 @@ func New(options *Options) (*Runner, error) { } httpxOptions := httpx.DefaultOptions + httpxOptions.Trace = options.Trace var np *networkpolicy.NetworkPolicy if options.Networkpolicy != nil { @@ -357,7 +361,8 @@ func New(options *Options) (*Runner, error) { runner.HostErrorsCache = gc } - runner.errorPageClassifier = errorpageclassifier.New() + runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build() + runner.pageTypeClassifier = pagetypeclassifier.New() if options.HttpApiEndpoint != "" { apiServer := NewServer(options.HttpApiEndpoint, options) @@ -437,7 +442,7 @@ func (r *Runner) prepareInput() { // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { - expandedTarget := r.countTargetFromRawTarget(target) + expandedTarget, _ := r.countTargetFromRawTarget(target) if expandedTarget > 0 { numHosts += expandedTarget r.hm.Set(target, nil) //nolint @@ -513,6 +518,24 @@ func (r *Runner) seen(k string) bool { return ok } +func (r *Runner) duplicate(result *Result) bool { + respSimHash := simhash.Simhash(simhash.NewWordFeatureSet(converstionutil.Bytes(result.Raw))) + if r.simHashes.Has(respSimHash) { + gologger.Debug().Msgf("Skipping duplicate response with simhash %d for URL %s\n", respSimHash, result.URL) + return true + } + + for simHash := range r.simHashes.GetALL(false) { + // lower threshold for increased precision + if simhash.Compare(simHash, respSimHash) <= 3 { + gologger.Debug().Msgf("Skipping near-duplicate response with simhash %d for URL %s\n", respSimHash, result.URL) + return true + } + } + _ = r.simHashes.Set(respSimHash, struct{}{}) + return false +} + func (r *Runner) testAndSet(k string) bool { // skip empty lines k = strings.TrimSpace(k) @@ -580,7 +603,7 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { for scanner.Scan() { target := strings.TrimSpace(scanner.Text()) // Used just to get the exact number of targets - expandedTarget := r.countTargetFromRawTarget(target) + expandedTarget, _ := r.countTargetFromRawTarget(target) if expandedTarget > 0 { numTargets += expandedTarget r.hm.Set(target, nil) //nolint @@ -590,12 +613,12 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { return numTargets, err } -func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) { +func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err error) { if rawTarget == "" { - return 0 + return 0, nil } if _, ok := r.hm.Get(rawTarget); ok { - return 0 + return 0, nil } expandedTarget := 0 @@ -605,14 +628,17 @@ func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) { expandedTarget = int(ipsCount) } case asn.IsASN(rawTarget): - cidrs, _ := asn.GetCIDRsForASNNum(rawTarget) + cidrs, err := asn.GetCIDRsForASNNum(rawTarget) + if err != nil { + return 0, err + } for _, cidr := range cidrs { expandedTarget += int(mapcidr.AddressCountIpnet(cidr)) } default: expandedTarget = 1 } - return expandedTarget + return expandedTarget, nil } var ( @@ -853,7 +879,7 @@ func (r *Runner) RunEnumeration() { continue } - if indexFile != nil { + if indexFile != nil && resp.Err == nil { indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) _, _ = indexFile.WriteString(indexData) } @@ -883,6 +909,11 @@ func (r *Runner) RunEnumeration() { logFilteredErrorPage(r.options.OutputFilterErrorPagePath, resp.URL) continue } + + if r.options.FilterOutDuplicates && r.duplicate(&resp) { + continue + } + if len(r.options.filterStatusCode) > 0 && sliceutil.Contains(r.options.filterStatusCode, resp.StatusCode) { continue } @@ -950,6 +981,80 @@ func (r *Runner) RunEnumeration() { continue } + if r.options.OutputMatchResponseTime != "" { + filterOps := FilterOperator{flag: "-mrt, -match-response-time"} + operator, value, err := filterOps.Parse(r.options.OutputMatchResponseTime) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) + switch operator { + // take negation of >= and > + case greaterThanEq, greaterThan: + if respTimeTaken < value { + continue + } + // take negation of <= and < + case lessThanEq, lessThan: + if respTimeTaken > value { + continue + } + // take negation of = + case equal: + if respTimeTaken != value { + continue + } + // take negation of != + case notEq: + if respTimeTaken == value { + continue + } + } + } + + if r.options.OutputFilterResponseTime != "" { + filterOps := FilterOperator{flag: "-frt, -filter-response-time"} + operator, value, err := filterOps.Parse(r.options.OutputFilterResponseTime) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) + switch operator { + case greaterThanEq: + if respTimeTaken >= value { + continue + } + case lessThanEq: + if respTimeTaken <= value { + continue + } + case equal: + if respTimeTaken == value { + continue + } + case lessThan: + if respTimeTaken < value { + continue + } + case greaterThan: + if respTimeTaken > value { + continue + } + case notEq: + if respTimeTaken != value { + continue + } + } + } + + if !r.options.DisableStdout && (!jsonOrCsv || jsonAndCsv || r.options.OutputAll) { + gologger.Silent().Msgf("%s\n", resp.str) + } + + if resp.Err != nil { + continue + } + // store responses or chain in directory URL, _ := urlutil.Parse(resp.URL) domainFile := resp.Method + ":" + URL.EscapedString() @@ -1016,71 +1121,6 @@ func (r *Runner) RunEnumeration() { _, _ = indexScreenshotFile.WriteString(indexData) } - if r.options.OutputMatchResponseTime != "" { - filterOps := FilterOperator{flag: "-mrt, -match-response-time"} - operator, value, err := filterOps.Parse(r.options.OutputMatchResponseTime) - if err != nil { - gologger.Fatal().Msg(err.Error()) - } - respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) - switch operator { - // take negation of >= and > - case greaterThanEq, greaterThan: - if respTimeTaken < value { - continue - } - // take negation of <= and < - case lessThanEq, lessThan: - if respTimeTaken > value { - continue - } - // take negation of = - case equal: - if respTimeTaken != value { - continue - } - // take negation of != - case notEq: - if respTimeTaken == value { - continue - } - } - } - if r.options.OutputFilterResponseTime != "" { - filterOps := FilterOperator{flag: "-frt, -filter-response-time"} - operator, value, err := filterOps.Parse(r.options.OutputFilterResponseTime) - if err != nil { - gologger.Fatal().Msg(err.Error()) - } - respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) - switch operator { - case greaterThanEq: - if respTimeTaken >= value { - continue - } - case lessThanEq: - if respTimeTaken <= value { - continue - } - case equal: - if respTimeTaken == value { - continue - } - case lessThan: - if respTimeTaken < value { - continue - } - case greaterThan: - if respTimeTaken > value { - continue - } - case notEq: - if respTimeTaken != value { - continue - } - } - } - if r.scanopts.StoreVisionReconClusters { foundCluster := false pHash, _ := resp.KnowledgeBase["pHash"].(uint64) @@ -1102,10 +1142,6 @@ func (r *Runner) RunEnumeration() { } } - if !r.options.DisableStdout && (!jsonOrCsv || jsonAndCsv || r.options.OutputAll) { - gologger.Silent().Msgf("%s\n", resp.str) - } - //nolint:errcheck // this method needs a small refactor to reduce complexity if plainFile != nil { plainFile.WriteString(resp.str + "\n") @@ -1523,7 +1559,10 @@ retry: if err := URL.MergePath(scanopts.RequestURI, scanopts.Unsafe); err != nil { gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI) } - var req *retryablehttp.Request + var ( + req *retryablehttp.Request + ctx context.Context + ) if target.CustomIP != "" { var requestIP string if iputil.IsIPv6(target.CustomIP) { @@ -1531,11 +1570,11 @@ retry: } else { requestIP = target.CustomIP } - ctx := context.WithValue(context.Background(), fastdialer.IP, requestIP) - req, err = hp.NewRequestWithContext(ctx, method, URL.String()) + ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP) } else { - req, err = hp.NewRequest(method, URL.String()) + ctx = context.Background() } + req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { return Result{URL: URL.String(), Input: origInput, Err: err} } @@ -2144,7 +2183,7 @@ retry: var pHash uint64 if scanopts.Screenshot { var err error - screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, time.Duration(scanopts.ScreenshotTimeout)*time.Second, r.options.CustomHeaders) + screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, scanopts.ScreenshotTimeout, scanopts.ScreenshotIdle, r.options.CustomHeaders) if err != nil { gologger.Warning().Msgf("Could not take screenshot '%s': %s", fullURL, err) } else { @@ -2239,7 +2278,7 @@ retry: ScreenshotBytes: screenshotBytes, HeadlessBody: headlessBody, KnowledgeBase: map[string]interface{}{ - "PageType": r.errorPageClassifier.Classify(respData), + "PageType": r.pageTypeClassifier.Classify(respData), "pHash": pHash, }, TechnologyDetails: technologyDetails, @@ -2252,6 +2291,9 @@ retry: result.Fqdns = resp.BodyDomains.Fqdns result.Domains = resp.BodyDomains.Domains } + if r.options.Trace { + result.Trace = req.TraceInfo + } return result } diff --git a/runner/runner_test.go b/runner/runner_test.go index 08698b20..f456c1ad 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -7,6 +7,8 @@ import ( _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/mapcidr/asn" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/stretchr/testify/require" ) @@ -106,11 +108,17 @@ func TestRunner_asn_targets(t *testing.T) { for _, ip := range ips { expected = append(expected, httpx.Target{Host: ip}) } + + if _, err := asn.GetIPAddressesAsStream(input); err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") { + t.Skip("skipping asn test due to missing/invalid api key") + return + } + got := []httpx.Target{} for target := range r.targets(r.hp, input) { got = append(got, target) } - require.ElementsMatch(t, expected, got, "could not exepcted output") + require.ElementsMatch(t, expected, got, "could not get expected output") } func TestRunner_countTargetFromRawTarget(t *testing.T) { @@ -120,32 +128,41 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) { input := "example.com" expected := 1 - got := r.countTargetFromRawTarget(input) + got, err := r.countTargetFromRawTarget(input) + require.Nil(t, err, "could not count targets") require.Equal(t, expected, got, "got wrong output") input = "example.com" expected = 0 err = r.hm.Set(input, nil) require.Nil(t, err, "could not set value to hm") - got = r.countTargetFromRawTarget(input) + got, err = r.countTargetFromRawTarget(input) + require.Nil(t, err, "could not count targets") + require.Equal(t, expected, got, "got wrong output") + + input = "173.0.84.0/24" + expected = 256 + got, err = r.countTargetFromRawTarget(input) + require.Nil(t, err, "could not count targets") require.Equal(t, expected, got, "got wrong output") input = "" expected = 0 - got = r.countTargetFromRawTarget(input) + got, err = r.countTargetFromRawTarget(input) + require.Nil(t, err, "could not count targets") require.Equal(t, expected, got, "got wrong output") if os.Getenv("PDCP_API_KEY") != "" { input = "AS14421" expected = 256 - got = r.countTargetFromRawTarget(input) + got, err = r.countTargetFromRawTarget(input) + if err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") { + t.Skip("skipping asn test due to missing/invalid api key") + return + } + require.Nil(t, err, "could not count targets") require.Equal(t, expected, got, "got wrong output") } - - input = "173.0.84.0/24" - expected = 256 - got = r.countTargetFromRawTarget(input) - require.Equal(t, expected, got, "got wrong output") } func TestRunner_urlWithComma_targets(t *testing.T) { diff --git a/runner/types.go b/runner/types.go index da3b1cc6..7bdb3a9c 100644 --- a/runner/types.go +++ b/runner/types.go @@ -9,9 +9,10 @@ import ( "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/options" - "github.com/mitchellh/mapstructure" + mapstructure "github.com/go-viper/mapstructure/v2" "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" mapsutil "github.com/projectdiscovery/utils/maps" wappalyzer "github.com/projectdiscovery/wappalyzergo" @@ -32,74 +33,91 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` - Err error `json:"-" csv:"-"` - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` - Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - CDNType string `json:"cdn_type,omitempty" csv:"cdn_type"` - SNI string `json:"sni,omitempty" csv:"sni"` - Port string `json:"port,omitempty" csv:"port"` - Raw string `json:"-" csv:"-"` - URL string `json:"url,omitempty" csv:"url"` - Input string `json:"input,omitempty" csv:"input"` - Location string `json:"location,omitempty" csv:"location"` - Title string `json:"title,omitempty" csv:"title"` - str string - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"body"` - BodyPreview string `json:"body_preview,omitempty" csv:"body_preview"` - ContentType string `json:"content_type,omitempty" csv:"content_type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` - FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon"` - FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path"` - FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url"` - FinalURL string `json:"final_url,omitempty" csv:"final_url"` - ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header"` - RawHeaders string `json:"raw_header,omitempty" csv:"raw_header"` - Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"time,omitempty" csv:"time"` - JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash"` - ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` - A []string `json:"a,omitempty" csv:"a"` - AAAA []string `json:"aaaa,omitempty" csv:"aaaa"` - CNAMEs []string `json:"cname,omitempty" csv:"cname"` - Technologies []string `json:"tech,omitempty" csv:"tech"` - Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words"` - Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status_code" csv:"status_code"` - ContentLength int `json:"content_length" csv:"content_length"` - Failed bool `json:"failed" csv:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` - HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body"` - ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes"` - StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"` - ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"` - ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel"` - KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase"` - Resolvers []string `json:"resolvers,omitempty" csv:"resolvers"` - Fqdns []string `json:"body_fqdn,omitempty"` - Domains []string `json:"body_domains,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"` + ASN *AsnResponse `json:"asn,omitempty" csv:"asn" mapstructure:"asn"` + Err error `json:"-" csv:"-" mapstructure:"-"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" mapstructure:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"tls" mapstructure:"tls"` + Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"` + CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"` + SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"` + Port string `json:"port,omitempty" csv:"port" mapstructure:"port"` + Raw string `json:"-" csv:"-" mapstructure:"-"` + URL string `json:"url,omitempty" csv:"url" mapstructure:"url"` + Input string `json:"input,omitempty" csv:"input" mapstructure:"input"` + Location string `json:"location,omitempty" csv:"location" mapstructure:"location"` + Title string `json:"title,omitempty" csv:"title" mapstructure:"title"` + str string `mapstructure:"-"` + Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"` + Error string `json:"error,omitempty" csv:"error" mapstructure:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"` + ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"` + BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"` + ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"` + Method string `json:"method,omitempty" csv:"method" mapstructure:"method"` + Host string `json:"host,omitempty" csv:"host" mapstructure:"host"` + Path string `json:"path,omitempty" csv:"path" mapstructure:"path"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"` + FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"` + FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"` + FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"` + FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"` + ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"` + RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"` + Request string `json:"request,omitempty" csv:"request" mapstructure:"request"` + ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"` + JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"` + A []string `json:"a,omitempty" csv:"a" mapstructure:"a"` + AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"` + CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain" mapstructure:"chain"` + Words int `json:"words" csv:"words" mapstructure:"words"` + Lines int `json:"lines" csv:"lines" mapstructure:"lines"` + StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"` + ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"` + Failed bool `json:"failed" csv:"failed" mapstructure:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"` + HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"` + ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"` + StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"` + ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"` + ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"` + KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"` + Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"` + Fqdns []string `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"` + Domains []string `json:"body_domains,omitempty" mapstructure:"body_domains"` + TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"` + RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"` + Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` + FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` + Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"trace" mapstructure:"trace"` +} - // Internal Fields - TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-"` - RequestRaw []byte `json:"-" csv:"-"` - Response *httpx.Response `json:"-" csv:"-"` - FaviconData []byte `json:"-" csv:"-"` +type Trace struct { + GetConn time.Time `json:"get_conn,omitempty"` + GotConn time.Time `json:"got_conn,omitempty"` + PutIdleConn time.Time `json:"put_idle_conn,omitempty"` + GotFirstResponseByte time.Time `json:"got_first_response_byte,omitempty"` + Got100Continue time.Time `json:"got_100_continue,omitempty"` + DNSStart time.Time `json:"dns_start,omitempty"` + DNSDone time.Time `json:"dns_done,omitempty"` + ConnectStart time.Time `json:"connect_start,omitempty"` + ConnectDone time.Time `json:"connect_done,omitempty"` + TLSHandshakeStart time.Time `json:"tls_handshake_start,omitempty"` + TLSHandshakeDone time.Time `json:"tls_handshake_done,omitempty"` + WroteHeaderField time.Time `json:"wrote_header_field,omitempty"` + WroteHeaders time.Time `json:"wrote_headers,omitempty"` + Wait100Continue time.Time `json:"wait_100_continue,omitempty"` + WroteRequest time.Time `json:"wrote_request,omitempty"` } // function to get dsl variables from result struct @@ -139,8 +157,7 @@ func evalDslExpr(result Result, dslExpr string) bool { func resultToMap(resp Result) (map[string]any, error) { m := make(map[string]any) config := &mapstructure.DecoderConfig{ - TagName: "json", - Result: &m, + Result: &m, } decoder, err := mapstructure.NewDecoder(config) if err != nil {