From 8773308b854d054f94aa2aff3726a32d55e7bd13 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sat, 25 Jan 2025 04:44:46 +0900 Subject: [PATCH 001/132] add digitalyama source to passive scraping --- v2/pkg/passive/sources.go | 2 + v2/pkg/passive/sources_test.go | 2 + .../sources/digitalyama/digitalyama.go | 128 ++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 v2/pkg/subscraping/sources/digitalyama/digitalyama.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index b8ce15472..5bc1b7e62 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -47,6 +47,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -94,6 +95,7 @@ var AllSources = [...]subscraping.Source{ // &reconcloud.Source{}, // failing due to cloudflare bot protection &builtwith.Source{}, &hudsonrock.Source{}, + &digitalyama.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 3223117e3..d6389acbd 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -54,6 +54,7 @@ var ( // "reconcloud", "builtwith", "hudsonrock", + "digitalyama" } expectedDefaultSources = []string{ @@ -89,6 +90,7 @@ var ( // "threatminer", // "reconcloud", "builtwith", + "digitalyama" } expectedDefaultRecursiveSources = []string{ diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go new file mode 100644 index 000000000..aa708ec25 --- /dev/null +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -0,0 +1,128 @@ +package digitalyama + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +type digitalYamaResponse struct { + Query string `json:"query"` + Count int `json:"count"` + Subdomains []string `json:"subdomains"` + UsageSummary struct { + QueryCost float64 `json:"query_cost"` + CreditsRemaining float64 `json:"credits_remaining"` + } `json:"usage_summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) + resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + var errResponse struct { + Detail []struct { + Loc []string `json:"loc"` + Msg string `json:"msg"` + Type string `json:"type"` + } `json:"detail"` + } + err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + s.errors++ + return + } + if len(errResponse.Detail) > 0 { + errMsg := errResponse.Detail[0].Msg + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)} + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + } + s.errors++ + return + } + + var response digitalYamaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range response.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "digitalyama" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} \ No newline at end of file From 71b884d2760e5de29e497d3a9fca0f4692ccd4d7 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sat, 25 Jan 2025 22:56:17 +0900 Subject: [PATCH 002/132] fix: correct syntax for digitalyama in sources_test.go --- v2/pkg/passive/sources_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index d6389acbd..a648b76c7 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -54,7 +54,7 @@ var ( // "reconcloud", "builtwith", "hudsonrock", - "digitalyama" + "digitalyama", } expectedDefaultSources = []string{ @@ -90,7 +90,7 @@ var ( // "threatminer", // "reconcloud", "builtwith", - "digitalyama" + "digitalyama", } expectedDefaultRecursiveSources = []string{ From ad8eb0ce7581405119ab57edc843213bef8dc546 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sun, 26 Jan 2025 07:54:40 +0900 Subject: [PATCH 003/132] fix: add missing newline at end of digitalyama.go --- v2/pkg/subscraping/sources/digitalyama/digitalyama.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go index aa708ec25..60027c71f 100644 --- a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + // jsoniter jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -20,11 +21,11 @@ type Source struct { } type digitalYamaResponse struct { - Query string `json:"query"` - Count int `json:"count"` - Subdomains []string `json:"subdomains"` + Query string `json:"query"` + Count int `json:"count"` + Subdomains []string `json:"subdomains"` UsageSummary struct { - QueryCost float64 `json:"query_cost"` + QueryCost float64 `json:"query_cost"` CreditsRemaining float64 `json:"credits_remaining"` } `json:"usage_summary"` } @@ -125,4 +126,4 @@ func (s *Source) Statistics() subscraping.Statistics { TimeTaken: s.timeTaken, Skipped: s.skipped, } -} \ No newline at end of file +} From a83d28a95883c0d7d1cd9d90ff2f037e6eda1773 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sun, 26 Jan 2025 07:56:35 +0900 Subject: [PATCH 004/132] test for workflow that RapidApi error in buildTest --- v2/pkg/subscraping/sources/digitalyama/digitalyama.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go index 60027c71f..eaa6f8420 100644 --- a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - // jsoniter jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" From 608f57913700b91fb221a267727f93d3e45202a7 Mon Sep 17 00:00:00 2001 From: Eunji Date: Sat, 18 Jan 2025 06:47:32 +0900 Subject: [PATCH 005/132] add threatcrowd --- v2/pkg/passive/sources.go | 2 + v2/pkg/passive/sources_test.go | 1 + .../sources/threatcrowd/threatcrowd.go | 117 ++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index b8ce15472..058c07d1f 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -43,6 +43,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" @@ -85,6 +86,7 @@ var AllSources = [...]subscraping.Source{ &shodan.Source{}, &sitedossier.Source{}, &threatbook.Source{}, + &threatcrowd.Source{}, &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 3223117e3..d1bb4cbc8 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -43,6 +43,7 @@ var ( "shodan", "sitedossier", "threatbook", + "threatcrowd", "virustotal", "waybackarchive", "whoisxmlapi", diff --git a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go new file mode 100644 index 000000000..cce9e05db --- /dev/null +++ b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -0,0 +1,117 @@ +package threatcrowd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// threatCrowdResponse represents the JSON response from the ThreatCrowd API. +type threatCrowdResponse struct { + ResponseCode string `json:"response_code"` + Subdomains []string `json:"subdomains"` + Undercount string `json:"undercount"` +} + +// Source implements the subscraping.Source interface for ThreatCrowd. +type Source struct { + timeTaken time.Duration + errors int + results int +} + +// Run queries the ThreatCrowd API for the given domain and returns found subdomains. +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func(startTime time.Time) { + defer func() { + s.timeTaken = time.Since(startTime) + close(results) + }() + + url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + resp, err := session.Client.Do(req) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} + s.errors++ + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + var tcResponse threatCrowdResponse + if err := json.Unmarshal(body, &tcResponse); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range tcResponse.Subdomains { + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + }(time.Now()) + + return results +} + +// Name returns the name of the source. +func (s *Source) Name() string { + return "threatcrowd" +} + +// IsDefault indicates whether this source is enabled by default. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport indicates if the source supports recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// NeedsKey indicates if the source requires an API key. +func (s *Source) NeedsKey() bool { + return false +} + +// AddApiKeys is a no-op since ThreatCrowd does not require an API key. +func (s *Source) AddApiKeys(_ []string) {} + +// Statistics returns usage statistics. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + } +} From 6cbfed905eea02b3b09bd3848e49ea26fc4c026e Mon Sep 17 00:00:00 2001 From: thurrsense Date: Mon, 3 Feb 2025 19:51:10 +0300 Subject: [PATCH 006/132] FIX: query for downloading subdomains in netlas.go --- v2/pkg/subscraping/sources/netlas/netlas.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/v2/pkg/subscraping/sources/netlas/netlas.go index 5df10c753..4c11a263d 100644 --- a/v2/pkg/subscraping/sources/netlas/netlas.go +++ b/v2/pkg/subscraping/sources/netlas/netlas.go @@ -98,10 +98,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Make a single POST request to get all domains via download method apiUrl := "https://app.netlas.io/api/domains/download/" - query := fmt.Sprintf("domain:(domain:*.%s AND NOT domain:%s)", domain, domain) + query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) requestBody := map[string]interface{}{ - "q": query, - "fields": []string{"*"}, + "q": query, + "fields": []string{"*"}, "source_type": "include", "size": domainsCount.Count, } @@ -116,8 +116,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ - "accept": "application/json", - "X-API-Key": randomApiKey, + "accept": "application/json", + "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} From 32dd39bea9e5f86b75be73574636a739a8b03eeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:26:07 +0000 Subject: [PATCH 007/132] chore(deps): bump github.com/projectdiscovery/gologger in /v2 Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.43 to 1.1.44. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.43...v1.1.44) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index cac91726d..89e93f883 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,7 @@ require ( github.com/projectdiscovery/chaos-client v0.5.2 github.com/projectdiscovery/dnsx v1.2.1 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.43 + github.com/projectdiscovery/gologger v1.1.44 github.com/projectdiscovery/ratelimit v0.0.70 github.com/projectdiscovery/retryablehttp-go v1.0.97 github.com/projectdiscovery/utils v0.4.9 diff --git a/v2/go.sum b/v2/go.sum index 4c9ece9b8..10a7f7011 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -202,8 +202,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= -github.com/projectdiscovery/gologger v1.1.43 h1:26DOeBUK2xus/UpM8jzHfNqEU5tWams3VGBtjJtI02I= -github.com/projectdiscovery/gologger v1.1.43/go.mod h1:993FxohnjVo34dSgE3bw+L4TOCDNQfQ5zNbK0YhYrEw= +github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= +github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= github.com/projectdiscovery/hmap v0.0.78 h1:eUjLjFR7KaxnlSIVQgT/Uc+i3EULGFb9Ax8qYAbbZno= github.com/projectdiscovery/hmap v0.0.78/go.mod h1:5iJ3+EtjRuechPw0W/9Mq5IDIMh68IBcIBEoLqS20NM= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= From 6914e519fd262786c76d6a5065737625ffdc53cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:32:09 +0000 Subject: [PATCH 008/132] chore(deps): bump github.com/projectdiscovery/dnsx in /v2 Bumps [github.com/projectdiscovery/dnsx](https://github.com/projectdiscovery/dnsx) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/projectdiscovery/dnsx/releases) - [Changelog](https://github.com/projectdiscovery/dnsx/blob/dev/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/dnsx/compare/v1.2.1...v1.2.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dnsx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 4 ++-- v2/go.sum | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 89e93f883..0ffb6fb19 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -8,7 +8,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.2 - github.com/projectdiscovery/dnsx v1.2.1 + github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.44 github.com/projectdiscovery/ratelimit v0.0.70 @@ -124,7 +124,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.64 + github.com/projectdiscovery/goflags v0.1.65 github.com/projectdiscovery/retryabledns v1.0.94 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 10a7f7011..f4fd203f6 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -194,14 +194,14 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= -github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg= -github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0= +github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= +github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk= github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= -github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= +github.com/projectdiscovery/goflags v0.1.65 h1:rjoj+5lP/FDzgeM0WILUTX9AOOnw0J0LXtl8P1SVeGE= +github.com/projectdiscovery/goflags v0.1.65/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g= github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= github.com/projectdiscovery/hmap v0.0.78 h1:eUjLjFR7KaxnlSIVQgT/Uc+i3EULGFb9Ax8qYAbbZno= @@ -224,8 +224,9 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= From 66430d4897462f7ee2cd78cc58dbf7219af8e1a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:38:57 +0000 Subject: [PATCH 009/132] chore(deps): bump github.com/projectdiscovery/hmap in /v2 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.78 to 0.0.79. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.78...v0.0.79) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0ffb6fb19..2cb42b880 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -72,7 +72,7 @@ require ( github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.1.0 // indirect github.com/projectdiscovery/fastdialer v0.3.0 // indirect - github.com/projectdiscovery/hmap v0.0.78 // indirect + github.com/projectdiscovery/hmap v0.0.79 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.1 // indirect github.com/refraction-networking/utls v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index f4fd203f6..0344621f7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -204,8 +204,8 @@ github.com/projectdiscovery/goflags v0.1.65 h1:rjoj+5lP/FDzgeM0WILUTX9AOOnw0J0LX github.com/projectdiscovery/goflags v0.1.65/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g= github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= -github.com/projectdiscovery/hmap v0.0.78 h1:eUjLjFR7KaxnlSIVQgT/Uc+i3EULGFb9Ax8qYAbbZno= -github.com/projectdiscovery/hmap v0.0.78/go.mod h1:5iJ3+EtjRuechPw0W/9Mq5IDIMh68IBcIBEoLqS20NM= +github.com/projectdiscovery/hmap v0.0.79 h1:tJsml7cHNFCaNdaJLh5p8yvwx/hlgFotdRRugX2tzRI= +github.com/projectdiscovery/hmap v0.0.79/go.mod h1:npHErVQb4U40KzQO87gqasTBbd8SRRIZ3nMAEqZN0Ic= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= From 16750c60caa4fafcae7203e3a1bc011e6ad30d00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:46:12 +0000 Subject: [PATCH 010/132] chore(deps): bump github.com/projectdiscovery/goflags in /v2 Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.64 to 0.1.71. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.64...v0.1.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 2cb42b880..8ef2f61df 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -124,7 +124,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.65 + github.com/projectdiscovery/goflags v0.1.71 github.com/projectdiscovery/retryabledns v1.0.94 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 0344621f7..bc0dcbddb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -200,8 +200,8 @@ github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6 github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.65 h1:rjoj+5lP/FDzgeM0WILUTX9AOOnw0J0LXtl8P1SVeGE= -github.com/projectdiscovery/goflags v0.1.65/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g= +github.com/projectdiscovery/goflags v0.1.71 h1:CmgHQUEo2VCUOypIsSvIa4YlpzIQSIg2bmfyQXYoe48= +github.com/projectdiscovery/goflags v0.1.71/go.mod h1:ikxJf0Jy7tQe13LpvTp0tanRAnqqYIlQlJaikSHnhY8= github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= github.com/projectdiscovery/hmap v0.0.79 h1:tJsml7cHNFCaNdaJLh5p8yvwx/hlgFotdRRugX2tzRI= From bc2d8d7b276ff9431ba2131f89a1d1be0036a347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 17 Feb 2025 16:29:08 +0300 Subject: [PATCH 011/132] update dnsrepo api endpoint --- v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go index 903f34985..b960883e0 100644 --- a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -40,7 +40,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.skipped = true return } - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", randomApiKey, domain)) + + randomApiInfo := strings.Split(randomApiKey, ":") + if len(randomApiInfo) != 2 { + s.skipped = true + return + } + + token := randomApiInfo[0] + apiKey := randomApiInfo[1] + + resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{" X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ From 2623ffd6efd2d72f49127439cbc7b9fd216567d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 17 Feb 2025 16:50:43 +0300 Subject: [PATCH 012/132] minor --- v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go index b960883e0..75cf859bd 100644 --- a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -21,7 +21,7 @@ type Source struct { } type DnsRepoResponse []struct { - Domain string + Domain string `json:"domain"` } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { @@ -50,7 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se token := randomApiInfo[0] apiKey := randomApiInfo[1] - resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{" X-API-Access": token}) + resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ From 262abb1297b69416994cd224de821abb18ea31cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:24:18 +0000 Subject: [PATCH 013/132] chore(deps): bump github.com/projectdiscovery/goflags in /v2 Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.71 to 0.1.72. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.71...v0.1.72) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 6 +++--- v2/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 8ef2f61df..83adfa75c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,8 +12,8 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.44 github.com/projectdiscovery/ratelimit v0.0.70 - github.com/projectdiscovery/retryablehttp-go v1.0.97 - github.com/projectdiscovery/utils v0.4.9 + github.com/projectdiscovery/retryablehttp-go v1.0.98 + github.com/projectdiscovery/utils v0.4.10 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -124,7 +124,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.71 + github.com/projectdiscovery/goflags v0.1.72 github.com/projectdiscovery/retryabledns v1.0.94 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index bc0dcbddb..f417183a1 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -200,8 +200,8 @@ github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6 github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.71 h1:CmgHQUEo2VCUOypIsSvIa4YlpzIQSIg2bmfyQXYoe48= -github.com/projectdiscovery/goflags v0.1.71/go.mod h1:ikxJf0Jy7tQe13LpvTp0tanRAnqqYIlQlJaikSHnhY8= +github.com/projectdiscovery/goflags v0.1.72 h1:tSR+BnfDLbfTGYYVg4k1oQcFOoYXPY1pllV0MHtx3ek= +github.com/projectdiscovery/goflags v0.1.72/go.mod h1:C2cZ+PJRx7bbArEp/qFUixjsYFDd3etFNNHMUdJqfr8= github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= github.com/projectdiscovery/hmap v0.0.79 h1:tJsml7cHNFCaNdaJLh5p8yvwx/hlgFotdRRugX2tzRI= @@ -214,10 +214,10 @@ github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0 github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM= github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= -github.com/projectdiscovery/retryablehttp-go v1.0.97 h1:6nee/vJjiZP3vOhyqLcpSADM3vqmcC2QOvaMIo+dKWQ= -github.com/projectdiscovery/retryablehttp-go v1.0.97/go.mod h1:ZvwB6IsIHf0YlovcEQufZ6OTluyWfxRd360SrKd9fPk= -github.com/projectdiscovery/utils v0.4.9 h1:GzYKy5iiCWEZZPGxrtgTOnRTZYiIAiCditGufp0nhGU= -github.com/projectdiscovery/utils v0.4.9/go.mod h1:/68d0OHGgYF4aW4X7kS1qlFlYOnZxgtFDN85iH732JI= +github.com/projectdiscovery/retryablehttp-go v1.0.98 h1:7wNj+vmx0z9FaQhtX/YRAMv4/1idj+bmFF07QvvEnIo= +github.com/projectdiscovery/retryablehttp-go v1.0.98/go.mod h1:ZS4sDlqTP2YbydUcjqXECdb3AIFvrT466OvcZjN3GlY= +github.com/projectdiscovery/utils v0.4.10 h1:rwTHowpQgEWZqpuKCzNP/loUNVcM0z3zyfjd8rvJRiM= +github.com/projectdiscovery/utils v0.4.10/go.mod h1:rjMHKcVQ0EbF6Zo69bjkDSqQHoXqaW/DxA8V9SU4/Zw= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From b9d4e9f627502009917902f6de0bdaf424fee5dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:30:33 +0000 Subject: [PATCH 014/132] chore(deps): bump github.com/projectdiscovery/utils in /v2 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.4.9 to 0.4.11. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.9...v0.4.11) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 4 ++-- v2/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 83adfa75c..4d4f622fb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,8 +12,8 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.44 github.com/projectdiscovery/ratelimit v0.0.70 - github.com/projectdiscovery/retryablehttp-go v1.0.98 - github.com/projectdiscovery/utils v0.4.10 + github.com/projectdiscovery/retryablehttp-go v1.0.99 + github.com/projectdiscovery/utils v0.4.11 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 diff --git a/v2/go.sum b/v2/go.sum index f417183a1..f56260e33 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -214,10 +214,10 @@ github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0 github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM= github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= -github.com/projectdiscovery/retryablehttp-go v1.0.98 h1:7wNj+vmx0z9FaQhtX/YRAMv4/1idj+bmFF07QvvEnIo= -github.com/projectdiscovery/retryablehttp-go v1.0.98/go.mod h1:ZS4sDlqTP2YbydUcjqXECdb3AIFvrT466OvcZjN3GlY= -github.com/projectdiscovery/utils v0.4.10 h1:rwTHowpQgEWZqpuKCzNP/loUNVcM0z3zyfjd8rvJRiM= -github.com/projectdiscovery/utils v0.4.10/go.mod h1:rjMHKcVQ0EbF6Zo69bjkDSqQHoXqaW/DxA8V9SU4/Zw= +github.com/projectdiscovery/retryablehttp-go v1.0.99 h1:S+lQqo1ZnO5aoWsBV8HapGslJSaYVUII954SnH1RSjw= +github.com/projectdiscovery/retryablehttp-go v1.0.99/go.mod h1:8Mv9L9vjmam16garE6/dqLFkT0ZcfLNSo9O1zFBiPlE= +github.com/projectdiscovery/utils v0.4.11 h1:MWqCFxYINQPa4KWMRNah7W0N1COGRhqOpGVhiR/VaO0= +github.com/projectdiscovery/utils v0.4.11/go.mod h1:47tvqErksJELcxDBH8An2i9qvUe5E1qR7B72xxqiyqU= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From 0463b623ee200841d6b61899d7dc1368a3d9b9a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:30:39 +0000 Subject: [PATCH 015/132] chore(deps): bump github.com/projectdiscovery/hmap in /v2 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.79 to 0.0.80. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.79...v0.0.80) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 83adfa75c..3211cc459 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -72,7 +72,7 @@ require ( github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.1.0 // indirect github.com/projectdiscovery/fastdialer v0.3.0 // indirect - github.com/projectdiscovery/hmap v0.0.79 // indirect + github.com/projectdiscovery/hmap v0.0.80 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.1 // indirect github.com/refraction-networking/utls v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index f417183a1..8db5bade9 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -204,8 +204,8 @@ github.com/projectdiscovery/goflags v0.1.72 h1:tSR+BnfDLbfTGYYVg4k1oQcFOoYXPY1pl github.com/projectdiscovery/goflags v0.1.72/go.mod h1:C2cZ+PJRx7bbArEp/qFUixjsYFDd3etFNNHMUdJqfr8= github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= -github.com/projectdiscovery/hmap v0.0.79 h1:tJsml7cHNFCaNdaJLh5p8yvwx/hlgFotdRRugX2tzRI= -github.com/projectdiscovery/hmap v0.0.79/go.mod h1:npHErVQb4U40KzQO87gqasTBbd8SRRIZ3nMAEqZN0Ic= +github.com/projectdiscovery/hmap v0.0.80 h1:2PSo3qQNKanK6i6DF4NzsVEJANe6tMIBmBtxvF4AKK8= +github.com/projectdiscovery/hmap v0.0.80/go.mod h1:YmZ9qwtl7MDJYrpxJ+MEw4N4V58w18WGPsQHgpdIV0s= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= From 62fbcade735ce0ba9e6f75a46c6830a14445ad58 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 18 Feb 2025 18:18:58 +0300 Subject: [PATCH 016/132] skip non-200 HTTP responses for anubis (#1541) --- v2/pkg/subscraping/sources/anubis/anubis.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/v2/pkg/subscraping/sources/anubis/anubis.go index 31e31a4c2..8ab84a3d0 100644 --- a/v2/pkg/subscraping/sources/anubis/anubis.go +++ b/v2/pkg/subscraping/sources/anubis/anubis.go @@ -4,6 +4,7 @@ package anubis import ( "context" "fmt" + "net/http" "time" jsoniter "github.com/json-iterator/go" @@ -38,6 +39,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return + } + var subdomains []string err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) if err != nil { From cf2b9d7686a0697005a58bb15c1243d26e53e10b Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 20 Feb 2025 23:52:44 +0300 Subject: [PATCH 017/132] fix virustotal limit err (#1548) --- v2/pkg/subscraping/sources/virustotal/virustotal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/v2/pkg/subscraping/sources/virustotal/virustotal.go index f996a1628..254965008 100644 --- a/v2/pkg/subscraping/sources/virustotal/virustotal.go +++ b/v2/pkg/subscraping/sources/virustotal/virustotal.go @@ -51,7 +51,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var cursor string = "" for { - var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain) + var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } From aa7fd52c9b07e3395522754cb6d4e6214e510597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 25 Feb 2025 16:24:55 +0300 Subject: [PATCH 018/132] update version --- v2/pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 1e9af43bc..8d2cda471 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.6.8` +const version = `v2.7.0` // showBanner is used to show the banner to the user func showBanner() { From a3f64602b678971ff145e65490e11bf3d9493a55 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:24:19 +0530 Subject: [PATCH 019/132] grouped dev --- .github/dependabot.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4ff8f26cb..ba33d912a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,10 +15,13 @@ updates: commit-message: prefix: "chore" include: "scope" - labels: - - "Type: Maintenance" allow: - dependency-name: "github.com/projectdiscovery/*" + groups: + modules: + patterns: ["github.com/projectdiscovery/*"] + labels: + - "Type: Maintenance" # # Maintain dependencies for GitHub Actions # - package-ecosystem: "github-actions" From b974fbdd74f329da9198cbd205a8411b759a4e3a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 31 Mar 2025 18:45:44 +0700 Subject: [PATCH 020/132] ci: adds compatibility checks also merge lint to build test Signed-off-by: Dwi Siswanto --- .github/workflows/build-test.yml | 30 +++++++++++++-------- .github/workflows/compatibility-checks.yaml | 18 +++++++++++++ .github/workflows/lint-test.yml | 24 ----------------- 3 files changed, 37 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/compatibility-checks.yaml delete mode 100644 .github/workflows/lint-test.yml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 02e1df74a..917b97779 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,24 +13,32 @@ on: type: boolean default: false -jobs: +jobs: + lint: + name: Lint Test + if: "${{ !endsWith(github.actor, '[bot]') }}" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v5 + with: + version: latest + args: --timeout 5m + working-directory: v2/ + build: name: Test Builds + needs: [lint] runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-13] steps: - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - - name: Check out code - uses: actions/checkout@v3 - - - name: Build - run: go build ./... + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go@v1 + - run: go build ./... working-directory: v2/ - name: Run tests diff --git a/.github/workflows/compatibility-checks.yaml b/.github/workflows/compatibility-checks.yaml new file mode 100644 index 000000000..329328641 --- /dev/null +++ b/.github/workflows/compatibility-checks.yaml @@ -0,0 +1,18 @@ +name: ♾️ Compatibility Checks + +on: + pull_request: + types: [opened, synchronize] + branches: + - dev + +jobs: + check: + if: github.actor == 'dependabot[bot]' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: projectdiscovery/actions/setup/go/compatibility-checks@master + diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml deleted file mode 100644 index da4f58218..000000000 --- a/.github/workflows/lint-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: πŸ™πŸ» Lint Test - -on: - pull_request: - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -jobs: - lint: - name: Lint Test - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.6.0 - with: - version: latest - args: --timeout 5m - working-directory: v2/ \ No newline at end of file From 232dbdef90d38a40df3ea879c6a6a648c4900b69 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 1 Apr 2025 15:04:37 +0700 Subject: [PATCH 021/132] ci(compat-checks): define go version file Signed-off-by: Dwi Siswanto --- .../{compatibility-checks.yaml => compat-checks.yaml} | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename .github/workflows/{compatibility-checks.yaml => compat-checks.yaml} (69%) diff --git a/.github/workflows/compatibility-checks.yaml b/.github/workflows/compat-checks.yaml similarity index 69% rename from .github/workflows/compatibility-checks.yaml rename to .github/workflows/compat-checks.yaml index 329328641..0b28347be 100644 --- a/.github/workflows/compatibility-checks.yaml +++ b/.github/workflows/compat-checks.yaml @@ -14,5 +14,7 @@ jobs: contents: write steps: - uses: actions/checkout@v4 - - uses: projectdiscovery/actions/setup/go/compatibility-checks@master + - uses: projectdiscovery/actions/setup/go/compat-checks@master + with: + go-version-file: 'v2/go.mod' From 44c1b3cf8b36e36e3b91f14bd1f8f77cdafdc954 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Apr 2025 08:56:06 +0000 Subject: [PATCH 022/132] chore(deps): bump golang.org/x/net from 0.33.0 to 0.36.0 in /v2 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.33.0 to 0.36.0. - [Commits](https://github.com/golang/net/compare/v0.33.0...v0.36.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- v2/go.mod | 13 +++++++------ v2/go.sum | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 38eefc5da..8154c5c0b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,7 @@ module github.com/projectdiscovery/subfinder/v2 go 1.21 +toolchain go1.24.1 require ( github.com/corpix/uarand v0.2.0 @@ -102,12 +103,12 @@ require ( github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.35.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.7 // indirect @@ -126,6 +127,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.72 github.com/projectdiscovery/retryabledns v1.0.94 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/net v0.36.0 // indirect + golang.org/x/sys v0.30.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index fb41ffc64..dfbd99cf6 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -334,8 +334,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -356,8 +356,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= @@ -367,8 +367,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -393,8 +393,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -402,8 +402,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -414,8 +414,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 97e35f63ca58ab7e2c7f4705296be54cca8cfb03 Mon Sep 17 00:00:00 2001 From: ghost Date: Fri, 11 Apr 2025 11:15:22 +0000 Subject: [PATCH 023/132] chore(deps): go mod tidy --- v2/go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index 8154c5c0b..f71a5dde0 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,7 @@ module github.com/projectdiscovery/subfinder/v2 -go 1.21 +go 1.23.0 + toolchain go1.24.1 require ( From b980e49f36e3387a473e07250fc3d3271d7ca80c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:12:30 +0000 Subject: [PATCH 024/132] chore(deps): bump the modules group in /v2 with 10 updates Bumps the modules group in /v2 with 10 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) | `1.1.44` | `1.1.53` | | [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.70` | `0.0.79` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.99` | `1.0.101` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.11` | `0.4.17` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.0` | `1.1.14` | | [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.3.0` | `0.4.0` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.80` | `0.0.81` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.1` | `0.1.12` | | [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) | `0.1.72` | `0.1.74` | | [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) | `1.0.94` | `1.0.98` | Updates `github.com/projectdiscovery/gologger` from 1.1.44 to 1.1.53 - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.44...v1.1.53) Updates `github.com/projectdiscovery/ratelimit` from 0.0.70 to 0.0.79 - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.70...v0.0.79) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.99 to 1.0.101 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.99...v1.0.101) Updates `github.com/projectdiscovery/utils` from 0.4.11 to 0.4.17 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.11...v0.4.17) Updates `github.com/projectdiscovery/cdncheck` from 1.1.0 to 1.1.14 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.0...v1.1.14) Updates `github.com/projectdiscovery/fastdialer` from 0.3.0 to 0.4.0 - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.3.0...v0.4.0) Updates `github.com/projectdiscovery/hmap` from 0.0.80 to 0.0.81 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.80...v0.0.81) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.1 to 0.1.12 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.1...v0.1.12) Updates `github.com/projectdiscovery/goflags` from 0.1.72 to 0.1.74 - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.72...v0.1.74) Updates `github.com/projectdiscovery/retryabledns` from 1.0.94 to 1.0.98 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.94...v1.0.98) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-version: 1.1.53 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/ratelimit dependency-version: 0.0.79 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.0.101 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.4.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.1.14 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/fastdialer dependency-version: 0.4.0 dependency-type: indirect update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.81 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.12 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/goflags dependency-version: 0.1.74 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.98 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- v2/go.mod | 35 ++++++++++++++-------------- v2/go.sum | 68 +++++++++++++++++++++++++++---------------------------- 2 files changed, 51 insertions(+), 52 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index f71a5dde0..163018466 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,14 +12,14 @@ require ( github.com/projectdiscovery/chaos-client v0.5.2 github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.44 - github.com/projectdiscovery/ratelimit v0.0.70 - github.com/projectdiscovery/retryablehttp-go v1.0.99 - github.com/projectdiscovery/utils v0.4.11 + github.com/projectdiscovery/gologger v1.1.53 + github.com/projectdiscovery/ratelimit v0.0.79 + github.com/projectdiscovery/retryablehttp-go v1.0.101 + github.com/projectdiscovery/utils v0.4.17 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 - golang.org/x/exp v0.0.0-20230420155640-133eef4313cb + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 gopkg.in/yaml.v3 v3.0.1 ) @@ -35,7 +35,6 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect @@ -46,7 +45,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaissmai/bart v0.9.5 // indirect + github.com/gaissmai/bart v0.17.10 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -72,11 +71,11 @@ require ( github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.0 // indirect - github.com/projectdiscovery/fastdialer v0.3.0 // indirect - github.com/projectdiscovery/hmap v0.0.80 // indirect + github.com/projectdiscovery/cdncheck v1.1.14 // indirect + github.com/projectdiscovery/fastdialer v0.4.0 // indirect + github.com/projectdiscovery/hmap v0.0.81 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.1 // indirect + github.com/projectdiscovery/networkpolicy v0.1.12 // indirect github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -85,7 +84,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -105,13 +104,13 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.35.0 // indirect - golang.org/x/mod v0.17.0 // indirect + golang.org/x/mod v0.22.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/sync v0.11.0 // indirect golang.org/x/term v0.29.0 // indirect golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.29.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect @@ -121,13 +120,13 @@ require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/miekg/dns v1.1.56 // indirect + github.com/miekg/dns v1.1.62 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.72 - github.com/projectdiscovery/retryabledns v1.0.94 // indirect + github.com/projectdiscovery/goflags v0.1.74 + github.com/projectdiscovery/retryabledns v1.0.98 // indirect golang.org/x/net v0.36.0 // indirect golang.org/x/sys v0.30.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index dfbd99cf6..a30939f4f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -69,8 +69,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= -github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= +github.com/gaissmai/bart v0.17.10 h1:TY1y++A6N/ESrwRLTRWrnVOrQpZqpOYSVnKMu/FYW6o= +github.com/gaissmai/bart v0.17.10/go.mod h1:JCPkH/Xt5bSPCKDc6OpzkhSCeib8BIxu3kthzZwcl6w= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -151,8 +151,8 @@ github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Cl github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= -github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -190,34 +190,34 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= -github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= +github.com/projectdiscovery/cdncheck v1.1.14 h1:CUeDkHkGavLUQ61UyJptSkchmGg5KwJI9bzsItsy3Q0= +github.com/projectdiscovery/cdncheck v1.1.14/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= -github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk= -github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= +github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w= +github.com/projectdiscovery/fastdialer v0.4.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.72 h1:tSR+BnfDLbfTGYYVg4k1oQcFOoYXPY1pllV0MHtx3ek= -github.com/projectdiscovery/goflags v0.1.72/go.mod h1:C2cZ+PJRx7bbArEp/qFUixjsYFDd3etFNNHMUdJqfr8= -github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= -github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= -github.com/projectdiscovery/hmap v0.0.80 h1:2PSo3qQNKanK6i6DF4NzsVEJANe6tMIBmBtxvF4AKK8= -github.com/projectdiscovery/hmap v0.0.80/go.mod h1:YmZ9qwtl7MDJYrpxJ+MEw4N4V58w18WGPsQHgpdIV0s= +github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= +github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= +github.com/projectdiscovery/gologger v1.1.53 h1:Er5nty/kifUDSr9MLgi8pzr0bveC+jco76Ittlg/AlM= +github.com/projectdiscovery/gologger v1.1.53/go.mod h1:PLaWBQIjfIaSAfAVAJ3MZctIyStGcI3CpaN7NLIejo8= +github.com/projectdiscovery/hmap v0.0.81 h1:M1wg+RS4xqNGCn0EjsjtrocUidfAk7iTztACYwCOe/M= +github.com/projectdiscovery/hmap v0.0.81/go.mod h1:zpx2eQHow57PNpgmVuhzcSGM1olHlMMBmqy2L5Z6FNk= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= -github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU= -github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0vd8ER+eCHQjc= -github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM= -github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= -github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= -github.com/projectdiscovery/retryablehttp-go v1.0.99 h1:S+lQqo1ZnO5aoWsBV8HapGslJSaYVUII954SnH1RSjw= -github.com/projectdiscovery/retryablehttp-go v1.0.99/go.mod h1:8Mv9L9vjmam16garE6/dqLFkT0ZcfLNSo9O1zFBiPlE= -github.com/projectdiscovery/utils v0.4.11 h1:MWqCFxYINQPa4KWMRNah7W0N1COGRhqOpGVhiR/VaO0= -github.com/projectdiscovery/utils v0.4.11/go.mod h1:47tvqErksJELcxDBH8An2i9qvUe5E1qR7B72xxqiyqU= +github.com/projectdiscovery/networkpolicy v0.1.12 h1:SwfCOm772jmkLQNKWKZHIhjJK3eYz4RVzMHZJfwtti8= +github.com/projectdiscovery/networkpolicy v0.1.12/go.mod h1:8fm26WaxgfNY3CGQWzohQy95oSzZlgikU9Oxd1Pq5mk= +github.com/projectdiscovery/ratelimit v0.0.79 h1:9Kzff7K5ZyAX0IWspx5X3fHtff0/TzFq7jDxEScO1Qw= +github.com/projectdiscovery/ratelimit v0.0.79/go.mod h1:z+hNaODlTmdajGj7V2yIqcQhB7fovdMSK2PNwpbrlHY= +github.com/projectdiscovery/retryabledns v1.0.98 h1:2rz0dExX6pJlp8BrF0ZwwimO+Y6T7KCDsstmUioF8cA= +github.com/projectdiscovery/retryabledns v1.0.98/go.mod h1:AeFHeqjpm375uKHKf9dn4+EvwsE/xXGGDU5cT5EEiqQ= +github.com/projectdiscovery/retryablehttp-go v1.0.101 h1:xmoXGVQ7DD/5YvDvtaOExbbF6aXlr5ARjssXgMdtkmY= +github.com/projectdiscovery/retryablehttp-go v1.0.101/go.mod h1:d+xU7CAHiOL/v+QQIHT4AXbEjTO7o0B5naQQOC0JDhw= +github.com/projectdiscovery/utils v0.4.17 h1:LATr3L0t+xNt+hLReQjEf66RpuALPk2vV9YV341XkNQ= +github.com/projectdiscovery/utils v0.4.17/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -251,8 +251,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= @@ -262,8 +262,8 @@ github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EU github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -336,12 +336,12 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -422,8 +422,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= From 5ddc497069d08af5c10fb401479f258b8f94f5f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 23:28:43 +0000 Subject: [PATCH 025/132] chore(deps): bump golang.org/x/net from 0.36.0 to 0.38.0 in /v2 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.36.0 to 0.38.0. - [Commits](https://github.com/golang/net/compare/v0.36.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.38.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- v2/go.mod | 12 ++++++------ v2/go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 163018466..fec9d8545 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -103,12 +103,12 @@ require ( github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.35.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.29.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -127,6 +127,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.74 github.com/projectdiscovery/retryabledns v1.0.98 // indirect - golang.org/x/net v0.36.0 // indirect - golang.org/x/sys v0.30.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index a30939f4f..1c8cfd42a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -334,8 +334,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -356,8 +356,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= @@ -367,8 +367,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -393,8 +393,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -402,8 +402,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -414,8 +414,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 9c1536361ca94b99eeff73c5e842e5f527cf5b0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Apr 2025 15:01:15 +0000 Subject: [PATCH 026/132] chore(deps): bump the modules group in /v2 with 5 updates Bumps the modules group in /v2 with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) | `1.1.53` | `1.1.54` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.101` | `1.0.109` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.17` | `0.4.18` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.14` | `1.1.15` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.81` | `0.0.87` | Updates `github.com/projectdiscovery/gologger` from 1.1.53 to 1.1.54 - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.53...v1.1.54) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.101 to 1.0.109 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.101...v1.0.109) Updates `github.com/projectdiscovery/utils` from 0.4.17 to 0.4.18 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.17...v0.4.18) Updates `github.com/projectdiscovery/cdncheck` from 1.1.14 to 1.1.15 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.14...v1.1.15) Updates `github.com/projectdiscovery/hmap` from 0.0.81 to 0.0.87 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.81...v0.0.87) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-version: 1.1.54 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.0.109 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.4.18 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.1.15 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.87 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- v2/go.mod | 37 +++++--- v2/go.sum | 272 +++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 260 insertions(+), 49 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index fec9d8545..53f8a8ed7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,10 +12,10 @@ require ( github.com/projectdiscovery/chaos-client v0.5.2 github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.53 + github.com/projectdiscovery/gologger v1.1.54 github.com/projectdiscovery/ratelimit v0.0.79 - github.com/projectdiscovery/retryablehttp-go v1.0.101 - github.com/projectdiscovery/utils v0.4.17 + github.com/projectdiscovery/retryablehttp-go v1.0.109 + github.com/projectdiscovery/utils v0.4.18 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -28,13 +28,17 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/STARRY-S/zip v0.2.1 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect - github.com/andybalholm/brotli v1.0.6 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/windows v1.0.1 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect @@ -43,7 +47,7 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaissmai/bart v0.17.10 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -54,26 +58,29 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mholt/archiver/v3 v3.5.1 // indirect + github.com/mholt/archives v0.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect - github.com/nwaples/rardecode v1.1.3 // indirect - github.com/pierrec/lz4/v4 v4.1.2 // indirect + github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.14 // indirect + github.com/projectdiscovery/cdncheck v1.1.15 // indirect github.com/projectdiscovery/fastdialer v0.4.0 // indirect - github.com/projectdiscovery/hmap v0.0.81 // indirect + github.com/projectdiscovery/hmap v0.0.87 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.12 // indirect github.com/refraction-networking/utls v1.6.7 // indirect @@ -81,7 +88,9 @@ require ( github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/therootcompany/xz v1.0.1 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect @@ -92,9 +101,8 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/weppos/publicsuffix-go v0.30.1 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -103,6 +111,7 @@ require ( github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 1c8cfd42a..e4b489d93 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,6 +1,25 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= @@ -8,6 +27,8 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2piv github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= +github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= @@ -18,9 +39,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46 github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -33,7 +53,14 @@ github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJR github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= +github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= +github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= +github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= +github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= +github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= @@ -44,6 +71,10 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99k github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= @@ -61,9 +92,11 @@ github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yA github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -71,19 +104,35 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gaissmai/bart v0.17.10 h1:TY1y++A6N/ESrwRLTRWrnVOrQpZqpOYSVnKMu/FYW6o= github.com/gaissmai/bart v0.17.10/go.mod h1:JCPkH/Xt5bSPCKDc6OpzkhSCeib8BIxu3kthzZwcl6w= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -99,30 +148,47 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= -github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= -github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -147,8 +213,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= -github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= +github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= +github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= @@ -166,9 +232,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= -github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= +github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -179,8 +244,8 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= -github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -190,8 +255,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.14 h1:CUeDkHkGavLUQ61UyJptSkchmGg5KwJI9bzsItsy3Q0= -github.com/projectdiscovery/cdncheck v1.1.14/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.15 h1:rRs3LW2MP7V8QeONVRYce6RhDcWp83O+AWmt+QQ4mBM= +github.com/projectdiscovery/cdncheck v1.1.15/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= @@ -202,10 +267,10 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.53 h1:Er5nty/kifUDSr9MLgi8pzr0bveC+jco76Ittlg/AlM= -github.com/projectdiscovery/gologger v1.1.53/go.mod h1:PLaWBQIjfIaSAfAVAJ3MZctIyStGcI3CpaN7NLIejo8= -github.com/projectdiscovery/hmap v0.0.81 h1:M1wg+RS4xqNGCn0EjsjtrocUidfAk7iTztACYwCOe/M= -github.com/projectdiscovery/hmap v0.0.81/go.mod h1:zpx2eQHow57PNpgmVuhzcSGM1olHlMMBmqy2L5Z6FNk= +github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= +github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= +github.com/projectdiscovery/hmap v0.0.87 h1:bSIqggL878qmmMG67rNgmEa314GB1o2rFM9wjJbsJHA= +github.com/projectdiscovery/hmap v0.0.87/go.mod h1:Je1MuSMaP1gM2toj/t3YQhGQpSJGYjQuQwHJpPyJT6g= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/networkpolicy v0.1.12 h1:SwfCOm772jmkLQNKWKZHIhjJK3eYz4RVzMHZJfwtti8= @@ -214,21 +279,24 @@ github.com/projectdiscovery/ratelimit v0.0.79 h1:9Kzff7K5ZyAX0IWspx5X3fHtff0/TzF github.com/projectdiscovery/ratelimit v0.0.79/go.mod h1:z+hNaODlTmdajGj7V2yIqcQhB7fovdMSK2PNwpbrlHY= github.com/projectdiscovery/retryabledns v1.0.98 h1:2rz0dExX6pJlp8BrF0ZwwimO+Y6T7KCDsstmUioF8cA= github.com/projectdiscovery/retryabledns v1.0.98/go.mod h1:AeFHeqjpm375uKHKf9dn4+EvwsE/xXGGDU5cT5EEiqQ= -github.com/projectdiscovery/retryablehttp-go v1.0.101 h1:xmoXGVQ7DD/5YvDvtaOExbbF6aXlr5ARjssXgMdtkmY= -github.com/projectdiscovery/retryablehttp-go v1.0.101/go.mod h1:d+xU7CAHiOL/v+QQIHT4AXbEjTO7o0B5naQQOC0JDhw= -github.com/projectdiscovery/utils v0.4.17 h1:LATr3L0t+xNt+hLReQjEf66RpuALPk2vV9YV341XkNQ= -github.com/projectdiscovery/utils v0.4.17/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= +github.com/projectdiscovery/retryablehttp-go v1.0.109 h1:z7USVuroBrJJH/ozGS4m+evwukFyJvIvjmvaTNXrhr8= +github.com/projectdiscovery/retryablehttp-go v1.0.109/go.mod h1:0A6WpqP585LzGIFHQButwfQin4746gvNK2BrGpmRoXI= +github.com/projectdiscovery/utils v0.4.18 h1:cSjMOLXI5gAajfA6KV+0iQG4dGx2IHWLQyND/Snvw7k= +github.com/projectdiscovery/utils v0.4.18/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= @@ -240,6 +308,8 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= +github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -250,11 +320,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= @@ -286,15 +359,14 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -320,10 +392,19 @@ github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21 github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -336,45 +417,98 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -404,7 +538,9 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -416,19 +552,76 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -441,6 +634,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -449,3 +643,11 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 8c54b9dbc09ce1389f415f91a9c814406bc44fcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:45:49 +0000 Subject: [PATCH 027/132] chore(deps): bump github.com/refraction-networking/utls in /v2 Bumps [github.com/refraction-networking/utls](https://github.com/refraction-networking/utls) from 1.6.7 to 1.7.0. - [Release notes](https://github.com/refraction-networking/utls/releases) - [Commits](https://github.com/refraction-networking/utls/compare/v1.6.7...v1.7.0) --- updated-dependencies: - dependency-name: github.com/refraction-networking/utls dependency-version: 1.7.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- v2/go.mod | 5 ++--- v2/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 53f8a8ed7..68a75aa6e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,7 +1,6 @@ module github.com/projectdiscovery/subfinder/v2 go 1.23.0 - toolchain go1.24.1 require ( @@ -43,7 +42,7 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.5.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -83,7 +82,7 @@ require ( github.com/projectdiscovery/hmap v0.0.87 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.12 // indirect - github.com/refraction-networking/utls v1.6.7 // indirect + github.com/refraction-networking/utls v1.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index e4b489d93..834f80d0f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -76,8 +76,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= +github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= @@ -284,8 +284,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.109/go.mod h1:0A6WpqP585LzGIFH github.com/projectdiscovery/utils v0.4.18 h1:cSjMOLXI5gAajfA6KV+0iQG4dGx2IHWLQyND/Snvw7k= github.com/projectdiscovery/utils v0.4.18/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= -github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= +github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= +github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= From 8b4bc843195085d345442b0e9cbe9cb0f7ed0b3d Mon Sep 17 00:00:00 2001 From: ghost Date: Wed, 23 Apr 2025 14:47:12 +0000 Subject: [PATCH 028/132] chore(deps): go mod tidy --- v2/go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index 68a75aa6e..cde73b596 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,7 @@ module github.com/projectdiscovery/subfinder/v2 -go 1.23.0 +go 1.24.0 + toolchain go1.24.1 require ( From 385e03eb95cc2fd999c1857a383ec93a8e893780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 28 Apr 2025 14:49:21 +0300 Subject: [PATCH 029/132] update version --- v2/pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 8d2cda471..2bb921d1b 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.7.0` +const version = `v2.7.1` // showBanner is used to show the banner to the user func showBanner() { From ab4e8852b7fc9525f4a60c7f96760f65e32c2713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 28 Apr 2025 15:10:46 +0300 Subject: [PATCH 030/132] update lint action --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 917b97779..0c9916b54 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v5 + uses: projectdiscovery/actions/golangci-lint/v2@v1 with: version: latest args: --timeout 5m From 678fbc811be3dde11f7948c864705413ccef26ff Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 29 Apr 2025 16:22:07 +0300 Subject: [PATCH 031/132] fix docker build (#1583) --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index eadf4f7e1..7d2291080 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.21-alpine AS build-env +FROM golang:1.24-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app @@ -8,7 +8,7 @@ RUN go mod download RUN go build ./cmd/subfinder # Release -FROM alpine:3.18.6 +FROM alpine:latest RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=build-env /app/v2/subfinder /usr/local/bin/ From 0c7381da1101df1522f4fe2ef9103e62db520c7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 14:19:44 +0000 Subject: [PATCH 032/132] chore(deps): bump the modules group across 1 directory with 7 updates Bumps the modules group with 5 updates in the /v2 directory: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.79` | `0.0.80` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.109` | `1.0.111` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.18` | `0.4.19` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.15` | `1.1.17` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.12` | `0.1.14` | Updates `github.com/projectdiscovery/ratelimit` from 0.0.79 to 0.0.80 - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.79...v0.0.80) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.109 to 1.0.111 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.109...v1.0.111) Updates `github.com/projectdiscovery/utils` from 0.4.18 to 0.4.19 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.18...v0.4.19) Updates `github.com/projectdiscovery/cdncheck` from 1.1.15 to 1.1.17 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.15...v1.1.17) Updates `github.com/projectdiscovery/hmap` from 0.0.87 to 0.0.88 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.87...v0.0.88) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.12 to 0.1.14 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.12...v0.1.14) Updates `github.com/projectdiscovery/retryabledns` from 1.0.98 to 1.0.99 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.98...v1.0.99) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-version: 0.0.80 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.0.111 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.4.19 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.1.17 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.88 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.14 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.99 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- v2/go.mod | 14 +++++++------- v2/go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index cde73b596..f7ac88ca7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -13,9 +13,9 @@ require ( github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.54 - github.com/projectdiscovery/ratelimit v0.0.79 - github.com/projectdiscovery/retryablehttp-go v1.0.109 - github.com/projectdiscovery/utils v0.4.18 + github.com/projectdiscovery/ratelimit v0.0.80 + github.com/projectdiscovery/retryablehttp-go v1.0.111 + github.com/projectdiscovery/utils v0.4.19 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -78,11 +78,11 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.15 // indirect + github.com/projectdiscovery/cdncheck v1.1.17 // indirect github.com/projectdiscovery/fastdialer v0.4.0 // indirect - github.com/projectdiscovery/hmap v0.0.87 // indirect + github.com/projectdiscovery/hmap v0.0.88 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.12 // indirect + github.com/projectdiscovery/networkpolicy v0.1.14 // indirect github.com/refraction-networking/utls v1.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -135,7 +135,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.98 // indirect + github.com/projectdiscovery/retryabledns v1.0.99 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 834f80d0f..5f2654e3c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -255,8 +255,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.15 h1:rRs3LW2MP7V8QeONVRYce6RhDcWp83O+AWmt+QQ4mBM= -github.com/projectdiscovery/cdncheck v1.1.15/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.17 h1:YSqKk05+UGSxmPIp7tlCvRegF63FUqO+mA2Wl/Je3gA= +github.com/projectdiscovery/cdncheck v1.1.17/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= @@ -269,20 +269,20 @@ github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.87 h1:bSIqggL878qmmMG67rNgmEa314GB1o2rFM9wjJbsJHA= -github.com/projectdiscovery/hmap v0.0.87/go.mod h1:Je1MuSMaP1gM2toj/t3YQhGQpSJGYjQuQwHJpPyJT6g= +github.com/projectdiscovery/hmap v0.0.88 h1:ygPNJOmw8Bc0Pu1+fa/foNBatSuSTwsUQLA8xBiCayI= +github.com/projectdiscovery/hmap v0.0.88/go.mod h1:ZrAtCoXdrYsYdlk/XslYFyuiDHkohmsQhvaLlN9cRRY= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.12 h1:SwfCOm772jmkLQNKWKZHIhjJK3eYz4RVzMHZJfwtti8= -github.com/projectdiscovery/networkpolicy v0.1.12/go.mod h1:8fm26WaxgfNY3CGQWzohQy95oSzZlgikU9Oxd1Pq5mk= -github.com/projectdiscovery/ratelimit v0.0.79 h1:9Kzff7K5ZyAX0IWspx5X3fHtff0/TzFq7jDxEScO1Qw= -github.com/projectdiscovery/ratelimit v0.0.79/go.mod h1:z+hNaODlTmdajGj7V2yIqcQhB7fovdMSK2PNwpbrlHY= -github.com/projectdiscovery/retryabledns v1.0.98 h1:2rz0dExX6pJlp8BrF0ZwwimO+Y6T7KCDsstmUioF8cA= -github.com/projectdiscovery/retryabledns v1.0.98/go.mod h1:AeFHeqjpm375uKHKf9dn4+EvwsE/xXGGDU5cT5EEiqQ= -github.com/projectdiscovery/retryablehttp-go v1.0.109 h1:z7USVuroBrJJH/ozGS4m+evwukFyJvIvjmvaTNXrhr8= -github.com/projectdiscovery/retryablehttp-go v1.0.109/go.mod h1:0A6WpqP585LzGIFHQButwfQin4746gvNK2BrGpmRoXI= -github.com/projectdiscovery/utils v0.4.18 h1:cSjMOLXI5gAajfA6KV+0iQG4dGx2IHWLQyND/Snvw7k= -github.com/projectdiscovery/utils v0.4.18/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= +github.com/projectdiscovery/networkpolicy v0.1.14 h1:XnwpGjF+h9xgwEIgrFG3G+7cGRPwh6FkxgQaLuw4rv4= +github.com/projectdiscovery/networkpolicy v0.1.14/go.mod h1:pat2rE4G7kbow8CQ/yOym0bdLPq8rj7ZZWn3/3OT4Rs= +github.com/projectdiscovery/ratelimit v0.0.80 h1:kDZ9Rgd/EiDR3fw8Ugtp4xVMaMZNzlEO8zCD4QholaE= +github.com/projectdiscovery/ratelimit v0.0.80/go.mod h1:UW6g3VZbX+wI6WLXsexWGpSYnaQ79Uv+VewRj2+pzXQ= +github.com/projectdiscovery/retryabledns v1.0.99 h1:DJ6TewgkwqJozDOPXhoOy/cdtuzJzbIQ/BFfWNYzHpw= +github.com/projectdiscovery/retryabledns v1.0.99/go.mod h1:pkPYuqtxhX6z1pYL+O6s4Lg7ubINt2jg12gsaW9O3kY= +github.com/projectdiscovery/retryablehttp-go v1.0.111 h1:HzkVN0IyC0RfVylBlgNoqaQgVvuUEvbDEVbpGyD/Y9M= +github.com/projectdiscovery/retryablehttp-go v1.0.111/go.mod h1:Tl6noELU9RpjwywMDw722HB038QXohl90g2ZtKSqbCI= +github.com/projectdiscovery/utils v0.4.19 h1:rWOOTWUMQK9gvgH01rrw0qFi0hrh712hM1pCUzapCqA= +github.com/projectdiscovery/utils v0.4.19/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= From 98e678907ac49a1036dd6f60fa6d318821d3d115 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 20 May 2025 16:30:06 +0300 Subject: [PATCH 033/132] Update installation doc (#1591) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97e0b09fa..4a3ce144e 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ OPTIMIZATION: # Installation -`subfinder` requires **go1.21** to install successfully. Run the following command to install the latest version: +`subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest From 7c89b4f77815c1482fafecfab3e53c1140217201 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 21:33:54 +0000 Subject: [PATCH 034/132] chore(deps): bump github.com/cloudflare/circl from 1.5.0 to 1.6.1 in /v2 Bumps [github.com/cloudflare/circl](https://github.com/cloudflare/circl) from 1.5.0 to 1.6.1. - [Release notes](https://github.com/cloudflare/circl/releases) - [Commits](https://github.com/cloudflare/circl/compare/v1.5.0...v1.6.1) --- updated-dependencies: - dependency-name: github.com/cloudflare/circl dependency-version: 1.6.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index f7ac88ca7..759e954b3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -43,7 +43,7 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.5.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 5f2654e3c..bf61a23ee 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -76,8 +76,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= -github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= From 5d75905d80db6ebbcef3b20124371ac4cdb1166f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 15:14:02 +0000 Subject: [PATCH 035/132] chore(deps): bump the modules group in /v2 with 6 updates Bumps the modules group in /v2 with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.80` | `0.0.81` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.111` | `1.0.112` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.17` | `1.1.18` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.88` | `0.0.89` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.14` | `0.1.15` | | [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) | `1.0.99` | `1.0.100` | Updates `github.com/projectdiscovery/ratelimit` from 0.0.80 to 0.0.81 - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.80...v0.0.81) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.111 to 1.0.112 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.111...v1.0.112) Updates `github.com/projectdiscovery/cdncheck` from 1.1.17 to 1.1.18 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.17...v1.1.18) Updates `github.com/projectdiscovery/hmap` from 0.0.88 to 0.0.89 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.88...v0.0.89) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.14 to 0.1.15 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.14...v0.1.15) Updates `github.com/projectdiscovery/retryabledns` from 1.0.99 to 1.0.100 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.99...v1.0.100) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-version: 0.0.81 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.0.112 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.1.18 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.89 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.15 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.100 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- v2/go.mod | 14 +++++++------- v2/go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 759e954b3..4980736f0 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -13,8 +13,8 @@ require ( github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.54 - github.com/projectdiscovery/ratelimit v0.0.80 - github.com/projectdiscovery/retryablehttp-go v1.0.111 + github.com/projectdiscovery/ratelimit v0.0.81 + github.com/projectdiscovery/retryablehttp-go v1.0.113 github.com/projectdiscovery/utils v0.4.19 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 @@ -49,7 +49,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaissmai/bart v0.17.10 // indirect + github.com/gaissmai/bart v0.20.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -78,11 +78,11 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.17 // indirect + github.com/projectdiscovery/cdncheck v1.1.20 // indirect github.com/projectdiscovery/fastdialer v0.4.0 // indirect - github.com/projectdiscovery/hmap v0.0.88 // indirect + github.com/projectdiscovery/hmap v0.0.89 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.14 // indirect + github.com/projectdiscovery/networkpolicy v0.1.15 // indirect github.com/refraction-networking/utls v1.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -135,7 +135,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.99 // indirect + github.com/projectdiscovery/retryabledns v1.0.100 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index bf61a23ee..cd165673c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -102,8 +102,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.17.10 h1:TY1y++A6N/ESrwRLTRWrnVOrQpZqpOYSVnKMu/FYW6o= -github.com/gaissmai/bart v0.17.10/go.mod h1:JCPkH/Xt5bSPCKDc6OpzkhSCeib8BIxu3kthzZwcl6w= +github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= +github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -255,8 +255,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.17 h1:YSqKk05+UGSxmPIp7tlCvRegF63FUqO+mA2Wl/Je3gA= -github.com/projectdiscovery/cdncheck v1.1.17/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.20 h1:qGbDCeOynz8IQczdQUm0VWd5OjUc1r5VPOnPrKnLSH8= +github.com/projectdiscovery/cdncheck v1.1.20/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= @@ -269,18 +269,18 @@ github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.88 h1:ygPNJOmw8Bc0Pu1+fa/foNBatSuSTwsUQLA8xBiCayI= -github.com/projectdiscovery/hmap v0.0.88/go.mod h1:ZrAtCoXdrYsYdlk/XslYFyuiDHkohmsQhvaLlN9cRRY= +github.com/projectdiscovery/hmap v0.0.89 h1:H+XIzk2YcE/9PpW/1N9NdQSrJWm2vthGPNIxSM+WHNU= +github.com/projectdiscovery/hmap v0.0.89/go.mod h1:N3gXFDLN6GqkYsk+2ZkReVOo32OBUV+PNiYyWhWG4ZE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.14 h1:XnwpGjF+h9xgwEIgrFG3G+7cGRPwh6FkxgQaLuw4rv4= -github.com/projectdiscovery/networkpolicy v0.1.14/go.mod h1:pat2rE4G7kbow8CQ/yOym0bdLPq8rj7ZZWn3/3OT4Rs= -github.com/projectdiscovery/ratelimit v0.0.80 h1:kDZ9Rgd/EiDR3fw8Ugtp4xVMaMZNzlEO8zCD4QholaE= -github.com/projectdiscovery/ratelimit v0.0.80/go.mod h1:UW6g3VZbX+wI6WLXsexWGpSYnaQ79Uv+VewRj2+pzXQ= -github.com/projectdiscovery/retryabledns v1.0.99 h1:DJ6TewgkwqJozDOPXhoOy/cdtuzJzbIQ/BFfWNYzHpw= -github.com/projectdiscovery/retryabledns v1.0.99/go.mod h1:pkPYuqtxhX6z1pYL+O6s4Lg7ubINt2jg12gsaW9O3kY= -github.com/projectdiscovery/retryablehttp-go v1.0.111 h1:HzkVN0IyC0RfVylBlgNoqaQgVvuUEvbDEVbpGyD/Y9M= -github.com/projectdiscovery/retryablehttp-go v1.0.111/go.mod h1:Tl6noELU9RpjwywMDw722HB038QXohl90g2ZtKSqbCI= +github.com/projectdiscovery/networkpolicy v0.1.15 h1:jHHPo43s/TSiWmm6T8kJuMqTwL3ukU92iQhxq0K0jg0= +github.com/projectdiscovery/networkpolicy v0.1.15/go.mod h1:GWMDGJmgJ9qGoVTUOxbq1oLIbEx0pPsL0VKlriCkn2g= +github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= +github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= +github.com/projectdiscovery/retryabledns v1.0.100 h1:u4dv88P4lZOUCBpCYtmd/McpJ9ThPmdgNLLNwK0a3g4= +github.com/projectdiscovery/retryabledns v1.0.100/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= +github.com/projectdiscovery/retryablehttp-go v1.0.113 h1:/FURBrwD1SquCvNUq8OoQJ7+krznzHZwgVoH8914IKw= +github.com/projectdiscovery/retryablehttp-go v1.0.113/go.mod h1:r0mNR7NnWWLrXoo9676hW6Gt6y4u9gPFdEbOUC+nqr4= github.com/projectdiscovery/utils v0.4.19 h1:rWOOTWUMQK9gvgH01rrw0qFi0hrh712hM1pCUzapCqA= github.com/projectdiscovery/utils v0.4.19/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From 9c5f8a8b17691c5d89fbc36ba135cf757b3c79ca Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Sat, 14 Jun 2025 03:15:23 -0700 Subject: [PATCH 036/132] Update GoReleaser configurations Signed-off-by: Emmanuel Ferdman --- v2/.goreleaser.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 6e97216d6..867267d2e 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -1,3 +1,5 @@ +version: 2 + before: hooks: - go mod tidy @@ -25,7 +27,8 @@ builds: main: cmd/subfinder/main.go archives: -- format: zip +- formats: + - zip name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: @@ -40,4 +43,4 @@ announce: discord: enabled: true - message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' From cd5d28901f45821b3dffe76f7964415b269aa920 Mon Sep 17 00:00:00 2001 From: Celesian <39219175+c3l3si4n@users.noreply.github.com> Date: Mon, 16 Jun 2025 14:14:44 -0300 Subject: [PATCH 037/132] Added pugrecon.com as a subdomains source (#1585) --- v2/pkg/passive/sources.go | 4 +- v2/pkg/passive/sources_test.go | 1 + v2/pkg/runner/options.go | 1 + .../subscraping/sources/pugrecon/pugrecon.go | 151 ++++++++++++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/subscraping/sources/pugrecon/pugrecon.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 7de378976..f6ebc2341 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" @@ -34,6 +35,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" @@ -47,7 +49,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -77,6 +78,7 @@ var AllSources = [...]subscraping.Source{ &netlas.Source{}, &leakix.Source{}, &quake.Source{}, + &pugrecon.Source{}, &rapiddns.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 848cc32e7..6b2875c3c 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -34,6 +34,7 @@ var ( "intelx", "netlas", "quake", + "pugrecon", "rapiddns", "redhuntlabs", // "riddler", // failing due to cloudfront protection diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 5e3d4b172..7f39e0315 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -244,6 +244,7 @@ func (options *Options) preProcessDomains() { var defaultRateLimits = []string{ "github=30/m", "fullhunt=60/m", + "pugrecon=10/s", fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", "shodan=1/s", diff --git a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go new file mode 100644 index 000000000..dca7e186d --- /dev/null +++ b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -0,0 +1,151 @@ +// Package pugrecon logic +package pugrecon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// pugreconResult stores a single result from the pugrecon API +type pugreconResult struct { + Name string `json:"name"` +} + +// pugreconAPIResponse stores the response from the pugrecon API +type pugreconAPIResponse struct { + Results []pugreconResult `json:"results"` + QuotaRemaining int `json:"quota_remaining"` + Limited bool `json:"limited"` + TotalResults int `json:"total_results"` + Message string `json:"message"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // Prepare POST request data + postData := map[string]string{"domain_name": domain} + bodyBytes, err := json.Marshal(postData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)} + s.errors++ + return + } + bodyReader := bytes.NewReader(bodyBytes) + + // Prepare headers + headers := map[string]string{ + "Authorization": "Bearer " + randomApiKey, + "Content-Type": "application/json", + "Accept": "application/json", + } + + apiURL := "https://pugrecon.com/api/v1/domains" + resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode) + // Attempt to read error message from body if possible + var apiResp pugreconAPIResponse + if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { + errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message) + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(errorMsg)} + s.errors++ + return + } + + var response pugreconAPIResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if response.Message != "" && !response.Limited { // Handle potential non-error messages, except rate limit info + // Log or handle message if needed, but don't treat as hard error unless necessary + } + + for _, subdomain := range response.Results { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name, + } + s.results++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "pugrecon" +} + +// IsDefault returns false as this is not a default source. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport returns false as this source does not support recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// NeedsKey returns true as this source requires an API key. +func (s *Source) NeedsKey() bool { + return true +} + +// AddApiKeys adds the API keys for the source. +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns the statistics for the source. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 423c732800282db22f223b4e35c2bd90f13820ab Mon Sep 17 00:00:00 2001 From: Pepijn van der Stap Date: Mon, 16 Jun 2025 19:16:56 +0200 Subject: [PATCH 038/132] Purge dead binaryedge source (#1592) * purge: binaryedge source * purge(binaryedge): omit dead source reference in passive --- v2/pkg/passive/sources.go | 2 - v2/pkg/passive/sources_test.go | 2 - .../sources/binaryedge/binaryedge.go | 194 ------------------ 3 files changed, 198 deletions(-) delete mode 100644 v2/pkg/subscraping/sources/binaryedge/binaryedge.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index f6ebc2341..481e0bf37 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -10,7 +10,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/builtwith" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" @@ -56,7 +55,6 @@ var AllSources = [...]subscraping.Source{ &alienvault.Source{}, &anubis.Source{}, &bevigil.Source{}, - &binaryedge.Source{}, &bufferover.Source{}, &c99.Source{}, &censys.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 6b2875c3c..3b42778c2 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -14,7 +14,6 @@ var ( "alienvault", "anubis", "bevigil", - "binaryedge", "bufferover", "c99", "censys", @@ -95,7 +94,6 @@ var ( expectedDefaultRecursiveSources = []string{ "alienvault", - "binaryedge", "bufferover", "certspotter", "crtsh", diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go deleted file mode 100644 index d798fe702..000000000 --- a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go +++ /dev/null @@ -1,194 +0,0 @@ -// Package binaryedge logic -package binaryedge - -import ( - "context" - "errors" - "fmt" - "math" - "net/url" - "strconv" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -const ( - v1 = "v1" - v2 = "v2" - baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" - v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" - v1PageSizeParam = "pagesize" - pageParam = "page" - firstPage = 1 - maxV1PageSize = 10000 -) - -type subdomainsResponse struct { - Message string `json:"message"` - Title string `json:"title"` - Status interface{} `json:"status"` // string for v1, int for v2 - Subdomains []string `json:"events"` - Page int `json:"page"` - PageSize int `json:"pagesize"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []string - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } - - var baseURL string - - authHeader := map[string]string{"X-Key": randomApiKey} - - if isV2(ctx, session, authHeader) { - baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) - } else { - authHeader = map[string]string{"X-Token": randomApiKey} - v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - baseURL = v1URLWithPageSize.String() - } - - if baseURL == "" { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL"), - } - s.errors++ - return - } - - s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) - }() - return results -} - -func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { - pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - - resp, err := session.Get(ctx, pageURL.String(), "", authHeader) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var response subdomainsResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - - // Check error messages - if response.Message != "" && response.Status != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New(response.Message)} - s.errors++ - return - } - - resp.Body.Close() - - for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - - totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) - nextPage := response.Page + 1 - if nextPage <= totalPages { - s.enumerate(ctx, session, baseURL, nextPage, authHeader, results) - } -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "binaryedge" -} - -func (s *Source) IsDefault() bool { - return false -} - -func (s *Source) HasRecursiveSupport() bool { - return true -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} - -func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { - resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) - if err != nil { - session.DiscardHTTPResponse(resp) - return false - } - - resp.Body.Close() - - return true -} - -func addURLParam(targetURL, name, value string) (*url.URL, error) { - u, err := url.Parse(targetURL) - if err != nil { - return u, err - } - q, _ := url.ParseQuery(u.RawQuery) - q.Add(name, value) - u.RawQuery = q.Encode() - - return u, nil -} From 3acf6d5349d39d010faa9fa5f55faf765e998147 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 22:52:25 +0530 Subject: [PATCH 039/132] chore(deps): bump the modules group in /v2 with 5 updates (#1597) Bumps the modules group in /v2 with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.113` | `1.0.114` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.19` | `0.4.20` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.20` | `1.1.23` | | [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.4.0` | `0.4.1` | | [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) | `1.0.100` | `1.0.101` | Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.113 to 1.0.114 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.113...v1.0.114) Updates `github.com/projectdiscovery/utils` from 0.4.19 to 0.4.20 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.19...v0.4.20) Updates `github.com/projectdiscovery/cdncheck` from 1.1.20 to 1.1.23 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.20...v1.1.23) Updates `github.com/projectdiscovery/fastdialer` from 0.4.0 to 0.4.1 - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.4.0...v0.4.1) Updates `github.com/projectdiscovery/retryabledns` from 1.0.100 to 1.0.101 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.100...v1.0.101) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.0.114 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.4.20 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.1.23 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/fastdialer dependency-version: 0.4.1 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.101 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 10 +++++----- v2/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 4980736f0..fdb011821 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,8 +14,8 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.54 github.com/projectdiscovery/ratelimit v0.0.81 - github.com/projectdiscovery/retryablehttp-go v1.0.113 - github.com/projectdiscovery/utils v0.4.19 + github.com/projectdiscovery/retryablehttp-go v1.0.114 + github.com/projectdiscovery/utils v0.4.20 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -78,8 +78,8 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.20 // indirect - github.com/projectdiscovery/fastdialer v0.4.0 // indirect + github.com/projectdiscovery/cdncheck v1.1.23 // indirect + github.com/projectdiscovery/fastdialer v0.4.1 // indirect github.com/projectdiscovery/hmap v0.0.89 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.15 // indirect @@ -135,7 +135,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.100 // indirect + github.com/projectdiscovery/retryabledns v1.0.101 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index cd165673c..c0de7afbd 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -255,14 +255,14 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.20 h1:qGbDCeOynz8IQczdQUm0VWd5OjUc1r5VPOnPrKnLSH8= -github.com/projectdiscovery/cdncheck v1.1.20/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.23 h1:LOd6Y7hnV6sXFBs4qGDM0N9xfheAmqLhsfH2cog+M2c= +github.com/projectdiscovery/cdncheck v1.1.23/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= -github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w= -github.com/projectdiscovery/fastdialer v0.4.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= +github.com/projectdiscovery/fastdialer v0.4.1 h1:kp6Q0odo0VZ0vZIGOn+q9aLgBSk6uYoD1MsjCAH8+h4= +github.com/projectdiscovery/fastdialer v0.4.1/go.mod h1:875Wlggf0JAz+fDIPwUQeeBqEF6nJA71XVrjuTZCV7I= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= @@ -277,12 +277,12 @@ github.com/projectdiscovery/networkpolicy v0.1.15 h1:jHHPo43s/TSiWmm6T8kJuMqTwL3 github.com/projectdiscovery/networkpolicy v0.1.15/go.mod h1:GWMDGJmgJ9qGoVTUOxbq1oLIbEx0pPsL0VKlriCkn2g= github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= -github.com/projectdiscovery/retryabledns v1.0.100 h1:u4dv88P4lZOUCBpCYtmd/McpJ9ThPmdgNLLNwK0a3g4= -github.com/projectdiscovery/retryabledns v1.0.100/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= -github.com/projectdiscovery/retryablehttp-go v1.0.113 h1:/FURBrwD1SquCvNUq8OoQJ7+krznzHZwgVoH8914IKw= -github.com/projectdiscovery/retryablehttp-go v1.0.113/go.mod h1:r0mNR7NnWWLrXoo9676hW6Gt6y4u9gPFdEbOUC+nqr4= -github.com/projectdiscovery/utils v0.4.19 h1:rWOOTWUMQK9gvgH01rrw0qFi0hrh712hM1pCUzapCqA= -github.com/projectdiscovery/utils v0.4.19/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= +github.com/projectdiscovery/retryabledns v1.0.101 h1:8DIVD8CL34Lc9h6KeOopPUfsPlcFxMlrnKOaI9VeOMk= +github.com/projectdiscovery/retryabledns v1.0.101/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= +github.com/projectdiscovery/retryablehttp-go v1.0.114 h1:JFvk7RJ2AUrHV9dScHcnyaBpQRGq1d8/QfrpccCT0xc= +github.com/projectdiscovery/retryablehttp-go v1.0.114/go.mod h1:ZXHlpbSw9w3nZqe1LH0GPX2UDAmv2QpUOoafy+xydYs= +github.com/projectdiscovery/utils v0.4.20 h1:7Fmjb+4YZJSzn7bL21sjF3wAR53eSi7VdAfDkDBUUwY= +github.com/projectdiscovery/utils v0.4.20/go.mod h1:RnC23+hI8j4drZFHQpMX92hV9++9d/yBeNr1pzcbF7Y= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= From b669adbfd2d30d26b3939d12ccd0bcb9487753c2 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 20 Jun 2025 19:32:13 +0300 Subject: [PATCH 040/132] modernize subfinder (#1601) * update gh lint action * modernize * minor * update lint * fix lint errs * fix MORE lint errs * join errs * fix MOOORE lint errs * add go version file * ai review * ignore anubis tests --- .github/workflows/build-test.yml | 14 +++++++++----- v2/pkg/passive/sources_wo_auth_test.go | 17 +++++++++-------- v2/pkg/resolve/resolve.go | 4 ++-- v2/pkg/runner/config.go | 6 +++++- v2/pkg/runner/outputter.go | 12 +++++++++--- v2/pkg/runner/runner.go | 12 +++++++++--- v2/pkg/subscraping/agent.go | 4 +++- .../sources/alienvault/alienvault.go | 4 ++-- v2/pkg/subscraping/sources/anubis/anubis.go | 6 +++--- v2/pkg/subscraping/sources/bevigil/bevigil.go | 4 ++-- .../sources/bufferover/bufferover.go | 4 ++-- .../subscraping/sources/builtwith/builtwith.go | 4 ++-- v2/pkg/subscraping/sources/c99/c99.go | 7 ++++++- v2/pkg/subscraping/sources/censys/censys.go | 4 ++-- .../sources/certspotter/certspotter.go | 8 ++++---- v2/pkg/subscraping/sources/chinaz/chinaz.go | 2 +- .../sources/commoncrawl/commoncrawl.go | 8 ++++---- v2/pkg/subscraping/sources/crtsh/crtsh.go | 15 ++++++++++----- .../sources/digitalyama/digitalyama.go | 7 ++++++- .../subscraping/sources/digitorus/digitorus.go | 7 ++++++- v2/pkg/subscraping/sources/dnsdb/dnsdb.go | 6 +++--- .../sources/dnsdumpster/dnsdumpster.go | 4 ++-- v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go | 2 +- v2/pkg/subscraping/sources/facebook/ctlogs.go | 8 ++++++-- .../subscraping/sources/facebook/ctlogs_test.go | 7 ++++++- v2/pkg/subscraping/sources/fofa/fofa.go | 6 +++--- v2/pkg/subscraping/sources/fullhunt/fullhunt.go | 4 ++-- v2/pkg/subscraping/sources/github/github.go | 8 ++++---- v2/pkg/subscraping/sources/gitlab/gitlab.go | 4 ++-- .../sources/hackertarget/hackertarget.go | 7 ++++++- .../sources/hudsonrock/hudsonrock.go | 4 ++-- v2/pkg/subscraping/sources/hunter/hunter.go | 6 +++--- v2/pkg/subscraping/sources/intelx/intelx.go | 10 +++++----- v2/pkg/subscraping/sources/leakix/leakix.go | 4 +++- v2/pkg/subscraping/sources/netlas/netlas.go | 16 +++++++++++++--- v2/pkg/subscraping/sources/pugrecon/pugrecon.go | 13 +++++++------ v2/pkg/subscraping/sources/quake/quake.go | 6 +++--- v2/pkg/subscraping/sources/rapiddns/rapiddns.go | 4 ++-- .../sources/reconcloud/reconcloud.go | 4 ++-- .../sources/redhuntlabs/redhuntlabs.go | 8 ++++---- v2/pkg/subscraping/sources/riddler/riddler.go | 2 +- v2/pkg/subscraping/sources/robtex/robtext.go | 4 ++-- .../sources/securitytrails/securitytrails.go | 6 +++--- v2/pkg/subscraping/sources/shodan/shodan.go | 2 +- .../sources/sitedossier/sitedossier.go | 4 ++-- .../sources/threatbook/threatbook.go | 4 ++-- .../sources/threatcrowd/threatcrowd.go | 7 ++++++- .../sources/threatminer/threatminer.go | 2 +- .../sources/virustotal/virustotal.go | 11 ++++++++--- .../sources/waybackarchive/waybackarchive.go | 2 +- .../sources/whoisxmlapi/whoisxmlapi.go | 4 ++-- v2/pkg/testutils/integration.go | 4 ++-- 52 files changed, 206 insertions(+), 126 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0c9916b54..54bac718b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -3,12 +3,12 @@ name: πŸ”¨ Build Test on: pull_request: paths: - - '**.go' - - '**.mod' + - "**.go" + - "**.mod" workflow_dispatch: inputs: short: - description: 'Use -short flag for tests' + description: "Use -short flag for tests" required: false type: boolean default: false @@ -21,8 +21,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 + with: + go-version-file: v2/go.mod - name: Run golangci-lint - uses: projectdiscovery/actions/golangci-lint/v2@v1 + uses: golangci/golangci-lint-action@v8 with: version: latest args: --timeout 5m @@ -38,6 +40,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 + with: + go-version-file: v2/go.mod - run: go build ./... working-directory: v2/ @@ -79,4 +83,4 @@ jobs: - name: Run Example run: go run . - working-directory: v2/examples \ No newline at end of file + working-directory: v2/examples diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index d5aef6530..036588cc3 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -24,14 +24,15 @@ func TestSourcesWithoutKeys(t *testing.T) { } ignoredSources := []string{ - "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - "riddler", // failing due to cloudfront protection - "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. - "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally - "waybackarchive", // Fails randomly - "alienvault", // 503 Service Temporarily Unavailable - "digitorus", // failing with "Failed to retrieve certificate" - "dnsdumpster", // failing with "unexpected status code 403 received" + "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ + "riddler", // failing due to cloudfront protection + "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally + "waybackarchive", // Fails randomly + "alienvault", // 503 Service Temporarily Unavailable + "digitorus", // failing with "Failed to retrieve certificate" + "dnsdumpster", // failing with "unexpected status code 403 received" + "anubis", // failing with "too many redirects" } domain := "hackerone.com" diff --git a/v2/pkg/resolve/resolve.go b/v2/pkg/resolve/resolve.go index 271d699fb..d7e4d887c 100644 --- a/v2/pkg/resolve/resolve.go +++ b/v2/pkg/resolve/resolve.go @@ -60,7 +60,7 @@ func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *Resoluti } go func() { - for i := 0; i < workers; i++ { + for range workers { resolutionPool.wg.Add(1) go resolutionPool.resolveWorker() } @@ -73,7 +73,7 @@ func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *Resoluti // InitWildcards inits the wildcard ips array func (r *ResolutionPool) InitWildcards(domain string) error { - for i := 0; i < maxWildcardChecks; i++ { + for range maxWildcardChecks { uid := xid.New().String() hosts, _ := r.DNSClient.Lookup(uid + "." + domain) diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 6c8ed2c6f..1751c9054 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -17,7 +17,11 @@ func createProviderConfigYAML(configFilePath string) error { if err != nil { return err } - defer configFile.Close() + defer func() { + if err := configFile.Close(); err != nil { + gologger.Error().Msgf("Error closing config file: %s", err) + } + }() sourcesRequiringApiKeysMap := make(map[string][]string) for _, source := range passive.AllSources { diff --git a/v2/pkg/runner/outputter.go b/v2/pkg/runner/outputter.go index faef0cae0..cfa4fc1a5 100644 --- a/v2/pkg/runner/outputter.go +++ b/v2/pkg/runner/outputter.go @@ -97,7 +97,9 @@ func writePlainHostIP(_ string, results map[string]resolve.Result, writer io.Wri _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() @@ -155,7 +157,9 @@ func writePlainHost(_ string, results map[string]resolve.HostEntry, writer io.Wr _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() @@ -228,7 +232,9 @@ func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, wr _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index ce9350419..f00ce6418 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -94,7 +94,9 @@ func (r *Runner) RunEnumerationWithCtx(ctx context.Context) error { return err } err = r.EnumerateMultipleDomainsWithCtx(ctx, f, outputs) - f.Close() + if closeErr := f.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", r.options.DomainsFile, closeErr) + } return err } @@ -139,7 +141,9 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) - file.Close() + if closeErr := file.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", r.options.OutputFile, closeErr) + } } else if r.options.OutputDirectory != "" { outputFile := path.Join(r.options.OutputDirectory, domain) if r.options.JSON { @@ -157,7 +161,9 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) - file.Close() + if closeErr := file.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", outputFile, closeErr) + } } else { _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers) } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 0e2b7f13a..4f2cab8c6 100644 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -119,7 +119,9 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) { gologger.Warning().Msgf("Could not discard response body: %s\n", err) return } - response.Body.Close() + if closeErr := response.Body.Close(); closeErr != nil { + gologger.Warning().Msgf("Could not close response body: %s\n", closeErr) + } } } diff --git a/v2/pkg/subscraping/sources/alienvault/alienvault.go b/v2/pkg/subscraping/sources/alienvault/alienvault.go index 857b2c78a..27e1f57b4 100644 --- a/v2/pkg/subscraping/sources/alienvault/alienvault.go +++ b/v2/pkg/subscraping/sources/alienvault/alienvault.go @@ -51,10 +51,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Error != "" { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/v2/pkg/subscraping/sources/anubis/anubis.go index 8ab84a3d0..6efbd5343 100644 --- a/v2/pkg/subscraping/sources/anubis/anubis.go +++ b/v2/pkg/subscraping/sources/anubis/anubis.go @@ -40,7 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } if resp.StatusCode != http.StatusOK { - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -49,11 +49,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range subdomains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} diff --git a/v2/pkg/subscraping/sources/bevigil/bevigil.go b/v2/pkg/subscraping/sources/bevigil/bevigil.go index 893db4913..2021e4c64 100644 --- a/v2/pkg/subscraping/sources/bevigil/bevigil.go +++ b/v2/pkg/subscraping/sources/bevigil/bevigil.go @@ -59,11 +59,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response.Subdomains) > 0 { subdomains = response.Subdomains diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/v2/pkg/subscraping/sources/bufferover/bufferover.go index bbfc884a3..207cb8190 100644 --- a/v2/pkg/subscraping/sources/bufferover/bufferover.go +++ b/v2/pkg/subscraping/sources/bufferover/bufferover.go @@ -69,11 +69,11 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) metaErrors := bufforesponse.Meta.Errors diff --git a/v2/pkg/subscraping/sources/builtwith/builtwith.go b/v2/pkg/subscraping/sources/builtwith/builtwith.go index 24f2e60cc..4c6b516f6 100644 --- a/v2/pkg/subscraping/sources/builtwith/builtwith.go +++ b/v2/pkg/subscraping/sources/builtwith/builtwith.go @@ -68,10 +68,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, result := range data.Results { for _, path := range result.Result.Paths { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)} diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go index bfd7fcf55..c2e45e818 100644 --- a/v2/pkg/subscraping/sources/c99/c99.go +++ b/v2/pkg/subscraping/sources/c99/c99.go @@ -56,7 +56,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) diff --git a/v2/pkg/subscraping/sources/censys/censys.go b/v2/pkg/subscraping/sources/censys/censys.go index 04ebe48e5..4427821ae 100644 --- a/v2/pkg/subscraping/sources/censys/censys.go +++ b/v2/pkg/subscraping/sources/censys/censys.go @@ -124,11 +124,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, hit := range censysResponse.Result.Hits { for _, name := range hit.Names { diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/v2/pkg/subscraping/sources/certspotter/certspotter.go index cf49d1dc9..e501e1730 100644 --- a/v2/pkg/subscraping/sources/certspotter/certspotter.go +++ b/v2/pkg/subscraping/sources/certspotter/certspotter.go @@ -59,10 +59,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, cert := range response { for _, subdomain := range cert.DNSNames { @@ -92,10 +92,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response) == 0 { break diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go index d7063973f..c6e43c05d 100644 --- a/v2/pkg/subscraping/sources/chinaz/chinaz.go +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -48,7 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := io.ReadAll(resp.Body) - resp.Body.Close() + session.DiscardHTTPResponse(resp) SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go index 34e50be9e..09ad4967d 100644 --- a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -59,13 +59,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) years := make([]string, 0) - for i := 0; i < maxYearsBack; i++ { + for i := range maxYearsBack { years = append(years, strconv.Itoa(year-i)) } @@ -155,7 +155,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se } } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) return true } } diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 4d008349f..76ac02b45 100644 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -14,6 +14,7 @@ import ( // postgres driver _ "github.com/lib/pq" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" contextutil "github.com/projectdiscovery/utils/context" ) @@ -60,7 +61,11 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio return 0 } - defer db.Close() + defer func() { + if closeErr := db.Close(); closeErr != nil { + gologger.Warning().Msgf("Could not close database connection: %s\n", closeErr) + } + }() limitClause := "" if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok { @@ -119,7 +124,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } count++ - for _, subdomain := range strings.Split(data, "\n") { + for subdomain := range strings.SplitSeq(data, "\n") { for _, value := range session.Extractor.Extract(subdomain) { if value != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} @@ -145,14 +150,14 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return false } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, subdomain := range subdomains { - for _, sub := range strings.Split(subdomain.NameValue, "\n") { + for sub := range strings.SplitSeq(subdomain.NameValue, "\n") { for _, value := range session.Extractor.Extract(sub) { if value != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go index eaa6f8420..7ab719576 100644 --- a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -54,7 +54,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() if resp.StatusCode != 200 { var errResponse struct { diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/v2/pkg/subscraping/sources/digitorus/digitorus.go index 6cf61dbcf..8c680a2b6 100644 --- a/v2/pkg/subscraping/sources/digitorus/digitorus.go +++ b/v2/pkg/subscraping/sources/digitorus/digitorus.go @@ -41,7 +41,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go index 2b18b8568..e5dc0e246 100644 --- a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -104,7 +104,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } else if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -113,7 +113,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -154,7 +154,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ } - resp.Body.Close() + session.DiscardHTTPResponse(resp) break } }() diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 2155e31cf..72be41c3b 100644 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -53,14 +53,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response response err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go index 75cf859bd..e98c82399 100644 --- a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -64,7 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) var result DnsRepoResponse err = json.Unmarshal(responseData, &result) if err != nil { diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs.go b/v2/pkg/subscraping/sources/facebook/ctlogs.go index 3f19af736..f96436008 100644 --- a/v2/pkg/subscraping/sources/facebook/ctlogs.go +++ b/v2/pkg/subscraping/sources/facebook/ctlogs.go @@ -45,7 +45,11 @@ func (k *apiKey) FetchAccessToken() { k.Error = err return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + gologger.Error().Msgf("error closing response body: %s", err) + } + }() bin, err := io.ReadAll(resp.Body) if err != nil { k.Error = err @@ -113,7 +117,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) response := &response{} if err := json.Unmarshal(bin, response); err != nil { s.errors++ diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go b/v2/pkg/subscraping/sources/facebook/ctlogs_test.go index 8869e7901..e9204055f 100644 --- a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go +++ b/v2/pkg/subscraping/sources/facebook/ctlogs_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/generic" ) @@ -41,7 +42,11 @@ func TestFacebookSource(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + gologger.Error().Msgf("error closing response body: %s", err) + } + }() bin, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go index 9a5294c75..78be69d1f 100644 --- a/v2/pkg/subscraping/sources/fofa/fofa.go +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -54,7 +54,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // fofa api doc https://fofa.info/static_pages/api_help - qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -68,10 +68,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Error { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go b/v2/pkg/subscraping/sources/fullhunt/fullhunt.go index 10c054e89..60c64e52d 100644 --- a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/v2/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -56,10 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range response.Hosts { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} s.results++ diff --git a/v2/pkg/subscraping/sources/github/github.go b/v2/pkg/subscraping/sources/github/github.go index 6034b2dbc..2e8b3224b 100644 --- a/v2/pkg/subscraping/sources/github/github.go +++ b/v2/pkg/subscraping/sources/github/github.go @@ -100,7 +100,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * if isForbidden && ratelimitRemaining == 0 { retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) tokens.setCurrentTokenExceeded(retryAfterSeconds) - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) } @@ -112,11 +112,11 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) if err != nil { @@ -173,7 +173,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r s.results++ } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) } // find subdomains in text matches diff --git a/v2/pkg/subscraping/sources/gitlab/gitlab.go b/v2/pkg/subscraping/sources/gitlab/gitlab.go index 9477b0bdf..902ef4691 100644 --- a/v2/pkg/subscraping/sources/gitlab/gitlab.go +++ b/v2/pkg/subscraping/sources/gitlab/gitlab.go @@ -74,7 +74,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var items []item err = jsoniter.NewDecoder(resp.Body).Decode(&items) @@ -114,7 +114,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * s.results++ } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) } defer wg.Done() }(it) diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go index de20cbbf0..c84957722 100644 --- a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -37,7 +37,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { diff --git a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go index b109a3192..51d6bc2fa 100644 --- a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -47,14 +47,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response hudsonrockResponse err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go index dc3b77021..fc2e88a11 100644 --- a/v2/pkg/subscraping/sources/hunter/hunter.go +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -59,7 +59,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var pages = 1 for currentPage := 1; currentPage <= pages; currentPage++ { // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 - qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -73,10 +73,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Code == 401 || response.Code == 400 { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/intelx/intelx.go b/v2/pkg/subscraping/sources/intelx/intelx.go index c08b3ff3c..b3a3a45a1 100644 --- a/v2/pkg/subscraping/sources/intelx/intelx.go +++ b/v2/pkg/subscraping/sources/intelx/intelx.go @@ -98,11 +98,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) status := 0 @@ -119,7 +119,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -127,10 +127,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) status = response.Status for _, hostname := range response.Selectors { diff --git a/v2/pkg/subscraping/sources/leakix/leakix.go b/v2/pkg/subscraping/sources/leakix/leakix.go index 7061ccd11..4159e6aba 100644 --- a/v2/pkg/subscraping/sources/leakix/leakix.go +++ b/v2/pkg/subscraping/sources/leakix/leakix.go @@ -46,7 +46,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + + defer session.DiscardHTTPResponse(resp) + if resp.StatusCode != 200 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} s.errors++ diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/v2/pkg/subscraping/sources/netlas/netlas.go index 4c11a263d..c29bc2c8b 100644 --- a/v2/pkg/subscraping/sources/netlas/netlas.go +++ b/v2/pkg/subscraping/sources/netlas/netlas.go @@ -77,7 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() body, err := io.ReadAll(resp.Body) if err != nil { @@ -99,7 +104,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se apiUrl := "https://app.netlas.io/api/domains/download/" query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) - requestBody := map[string]interface{}{ + requestBody := map[string]any{ "q": query, "fields": []string{"*"}, "source_type": "include", @@ -124,7 +129,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() body, err = io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} diff --git a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go index dca7e186d..e7f2255f8 100644 --- a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go +++ b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -78,7 +78,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to close response body: %w", err)} + s.errors++ + } + }() if resp.StatusCode != http.StatusOK { errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode) @@ -87,7 +92,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message) } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(errorMsg)} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", errorMsg)} s.errors++ return } @@ -100,10 +105,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - if response.Message != "" && !response.Limited { // Handle potential non-error messages, except rate limit info - // Log or handle message if needed, but don't treat as hard error unless necessary - } - for _, subdomain := range response.Results { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name, diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index 075d35e13..b5f6864e1 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -58,7 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // quake api doc https://quake.360.cn/quake/#/help - var requestBody = []byte(fmt.Sprintf(`{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)) + var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain) resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, }, bytes.NewReader(requestBody)) @@ -74,10 +74,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Code != 0 { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go index 5daae6153..89ec70632 100644 --- a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -48,11 +48,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go index 5a638b5f4..8aacc7d4e 100644 --- a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -56,10 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response.CloudAssetsList) > 0 { for _, cloudAsset := range response.CloudAssetsList { diff --git a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index ecbdcace2..a07e9a5b4 100644 --- a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -67,12 +67,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.errors++ return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Metadata.ResultCount > pageSize { totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize for page := 1; page <= totalPages; page++ { @@ -88,12 +88,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.errors++ continue } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, subdomain := range response.Subdomains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/v2/pkg/subscraping/sources/riddler/riddler.go index 0afe725eb..9a3401c27 100644 --- a/v2/pkg/subscraping/sources/riddler/riddler.go +++ b/v2/pkg/subscraping/sources/riddler/riddler.go @@ -48,7 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.results++ } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) }() return results diff --git a/v2/pkg/subscraping/sources/robtex/robtext.go b/v2/pkg/subscraping/sources/robtex/robtext.go index 5d130a42d..9214363ed 100644 --- a/v2/pkg/subscraping/sources/robtex/robtext.go +++ b/v2/pkg/subscraping/sources/robtex/robtext.go @@ -89,6 +89,8 @@ func enumerate(ctx context.Context, session *subscraping.Session, targetURL stri return results, err } + defer session.DiscardHTTPResponse(resp) + scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() @@ -104,8 +106,6 @@ func enumerate(ctx context.Context, session *subscraping.Session, targetURL stri results = append(results, response) } - resp.Body.Close() - return results, nil } diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go index e254d9c20..e6d38fe44 100644 --- a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -60,7 +60,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var err error if scrollId == "" { - var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain)) + var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { @@ -83,11 +83,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range securityTrailsResponse.Records { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index 8d4925d79..d2224a810 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -56,7 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go index dacc3c66b..38d31dd5c 100644 --- a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -63,10 +63,10 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba if err != nil { results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { diff --git a/v2/pkg/subscraping/sources/threatbook/threatbook.go b/v2/pkg/subscraping/sources/threatbook/threatbook.go index 1c10a8d10..0033a4d32 100644 --- a/v2/pkg/subscraping/sources/threatbook/threatbook.go +++ b/v2/pkg/subscraping/sources/threatbook/threatbook.go @@ -64,10 +64,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.ResponseCode != 0 { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go index cce9e05db..e09ab91ef 100644 --- a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -51,7 +51,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() if resp.StatusCode != http.StatusOK { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/v2/pkg/subscraping/sources/threatminer/threatminer.go index adffabd89..6776f9ef9 100644 --- a/v2/pkg/subscraping/sources/threatminer/threatminer.go +++ b/v2/pkg/subscraping/sources/threatminer/threatminer.go @@ -44,7 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/v2/pkg/subscraping/sources/virustotal/virustotal.go index 254965008..7ac4c6c50 100644 --- a/v2/pkg/subscraping/sources/virustotal/virustotal.go +++ b/v2/pkg/subscraping/sources/virustotal/virustotal.go @@ -49,9 +49,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if randomApiKey == "" { return } - var cursor string = "" + var cursor = "" for { - var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) + var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } @@ -62,7 +62,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go index 5edc78382..f3cba9809 100644 --- a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -39,7 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { diff --git a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index eafd340af..18375e3ff 100644 --- a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -67,11 +67,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range data.Result.Records { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index 2358cad20..d18230e95 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -27,8 +27,8 @@ func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]st return nil, err } var parts []string - items := strings.Split(string(data), "\n") - for _, i := range items { + items := strings.SplitSeq(string(data), "\n") + for i := range items { if i != "" { parts = append(parts, i) } From 7526dd8dd0227ea97a7b9f43f2a5f395477a9ade Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 20 Jun 2025 20:27:54 +0300 Subject: [PATCH 041/132] add RSECloud (#1604) * add RSECloud * discard after decoding --------- Co-authored-by: Sandeep Singh --- v2/pkg/passive/sources.go | 2 + v2/pkg/passive/sources_test.go | 2 + .../subscraping/sources/rsecloud/rsecloud.go | 117 ++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 v2/pkg/subscraping/sources/rsecloud/rsecloud.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 481e0bf37..e776c572a 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -39,6 +39,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" @@ -81,6 +82,7 @@ var AllSources = [...]subscraping.Source{ &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, + &rsecloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 3b42778c2..a63760296 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -38,6 +38,7 @@ var ( "redhuntlabs", // "riddler", // failing due to cloudfront protection "robtex", + "rsecloud", "securitytrails", "shodan", "sitedossier", @@ -79,6 +80,7 @@ var ( "redhuntlabs", "robtex", // "riddler", // failing due to cloudfront protection + "rsecloud", "securitytrails", "shodan", "virustotal", diff --git a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go new file mode 100644 index 000000000..c06361d67 --- /dev/null +++ b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -0,0 +1,117 @@ +package rsecloud + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Count int `json:"count"` + Data []string `json:"data"` + Page int `json:"page"` + PageSize int `json:"pagesize"` + TotalPages int `json:"total_pages"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "X-API-Key": randomApiKey} + + fetchSubdomains := func(endpoint string) { + page := 1 + for { + resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var rseCloudResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&rseCloudResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range rseCloudResponse.Data { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + + if page >= rseCloudResponse.TotalPages { + break + } + page++ + } + } + + fetchSubdomains("active") + fetchSubdomains("passive") + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "rsecloud" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From a0be3ff3eeee381939c5fd8331255764a1bb4a78 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 20 Jun 2025 21:41:19 +0300 Subject: [PATCH 042/132] Fix paginations (#1603) * fix quake pagination * update page param * skip empty * fix test * minor --- v2/pkg/subscraping/sources/hunter/hunter.go | 2 +- v2/pkg/subscraping/sources/quake/quake.go | 66 +++++++++++-------- .../subscraping/sources/rsecloud/rsecloud.go | 2 +- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go index fc2e88a11..960563279 100644 --- a/v2/pkg/subscraping/sources/hunter/hunter.go +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -60,7 +60,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for currentPage := 1; currentPage <= pages; currentPage++ { // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index b5f6864e1..5cac4ed2e 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -58,44 +58,58 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // quake api doc https://quake.360.cn/quake/#/help - var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain) - resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ - "Content-Type": "application/json", "X-QuakeToken": randomApiKey, - }, bytes.NewReader(requestBody)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } + var pageSize = 500 + var start = 0 + var totalResults = -1 + + for { + var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ + "Content-Type": "application/json", "X-QuakeToken": randomApiKey, + }, bytes.NewReader(requestBody)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } - var response quakeResults - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + var response quakeResults + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } session.DiscardHTTPResponse(resp) - return - } - session.DiscardHTTPResponse(resp) - if response.Code != 0 { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + if response.Code != 0 { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + } + s.errors++ + return + } + + if totalResults == -1 { + totalResults = response.Meta.Pagination.Total } - s.errors++ - return - } - if response.Meta.Pagination.Total > 0 { for _, quakeDomain := range response.Data { subdomain := quakeDomain.Service.HTTP.Host if strings.ContainsAny(subdomain, "ζš‚ζ— ζƒι™") { - subdomain = "" + continue } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } + + if len(response.Data) == 0 || start+pageSize >= totalResults { + break + } + + start += pageSize } }() diff --git a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go index c06361d67..8601b7732 100644 --- a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -96,7 +96,7 @@ func (s *Source) IsDefault() bool { } func (s *Source) HasRecursiveSupport() bool { - return true + return false } func (s *Source) NeedsKey() bool { From 7f3f2c2d5f192751f17bfae274e006e4cd669e91 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Fri, 20 Jun 2025 21:45:21 +0300 Subject: [PATCH 043/132] read key from env if provided (#1607) * read key from env if provided * remove debug code --- v2/pkg/passive/sources.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index e776c572a..20013042f 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -1,6 +1,8 @@ package passive import ( + "fmt" + "os" "strings" "golang.org/x/exp/maps" @@ -168,6 +170,15 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup } } + // TODO: Consider refactoring this to avoid potential duplication issues + for _, source := range sources { + if source.NeedsKey() { + if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" { + source.AddApiKeys([]string{apiKey}) + } + } + } + // Create the agent, insert the sources and remove the excluded sources agent := &Agent{sources: maps.Values(sources)} From 3d12dd7dfbf6ffbbb208b20990331336b6df9a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 20 Jun 2025 21:50:06 +0300 Subject: [PATCH 044/132] update version --- v2/pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 2bb921d1b..a0d3a7d1c 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.7.1` +const version = `v2.8.0` // showBanner is used to show the banner to the user func showBanner() { From a3674edd509695a4ee11313a768f003ca0da0da1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 16:38:50 +0000 Subject: [PATCH 045/132] chore(deps): bump the modules group in /v2 with 6 updates Bumps the modules group in /v2 with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.114` | `1.0.115` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.20` | `0.4.21` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.23` | `1.1.24` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.89` | `0.0.90` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.15` | `0.1.16` | | [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) | `1.0.101` | `1.0.102` | Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.114 to 1.0.115 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.114...v1.0.115) Updates `github.com/projectdiscovery/utils` from 0.4.20 to 0.4.21 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.20...v0.4.21) Updates `github.com/projectdiscovery/cdncheck` from 1.1.23 to 1.1.24 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.23...v1.1.24) Updates `github.com/projectdiscovery/hmap` from 0.0.89 to 0.0.90 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.89...v0.0.90) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.15 to 0.1.16 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.15...v0.1.16) Updates `github.com/projectdiscovery/retryabledns` from 1.0.101 to 1.0.102 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.101...v1.0.102) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.0.115 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.4.21 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.1.24 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.90 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.16 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.102 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] --- v2/go.mod | 18 +++++++++--------- v2/go.sum | 34 ++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index fdb011821..273a3a328 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,8 +14,8 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.54 github.com/projectdiscovery/ratelimit v0.0.81 - github.com/projectdiscovery/retryablehttp-go v1.0.114 - github.com/projectdiscovery/utils v0.4.20 + github.com/projectdiscovery/retryablehttp-go v1.0.115 + github.com/projectdiscovery/utils v0.4.21 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -45,7 +45,7 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect @@ -78,11 +78,11 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.23 // indirect + github.com/projectdiscovery/cdncheck v1.1.24 // indirect github.com/projectdiscovery/fastdialer v0.4.1 // indirect - github.com/projectdiscovery/hmap v0.0.89 // indirect + github.com/projectdiscovery/hmap v0.0.90 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.15 // indirect + github.com/projectdiscovery/networkpolicy v0.1.16 // indirect github.com/refraction-networking/utls v1.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -127,15 +127,15 @@ require ( require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.62 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.101 // indirect + github.com/projectdiscovery/retryabledns v1.0.102 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index c0de7afbd..80900dbb4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -84,12 +84,13 @@ github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= @@ -249,14 +250,15 @@ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.23 h1:LOd6Y7hnV6sXFBs4qGDM0N9xfheAmqLhsfH2cog+M2c= -github.com/projectdiscovery/cdncheck v1.1.23/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.24 h1:6pJ4XnovIrTWzlCJs5/QD1tv6wvK0wiICmmdY0/8WAs= +github.com/projectdiscovery/cdncheck v1.1.24/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= @@ -269,20 +271,20 @@ github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.89 h1:H+XIzk2YcE/9PpW/1N9NdQSrJWm2vthGPNIxSM+WHNU= -github.com/projectdiscovery/hmap v0.0.89/go.mod h1:N3gXFDLN6GqkYsk+2ZkReVOo32OBUV+PNiYyWhWG4ZE= +github.com/projectdiscovery/hmap v0.0.90 h1:p8HWGvPI88hgJoAb4ayR1Oo5VzqPrOCdFG7mASUhQI4= +github.com/projectdiscovery/hmap v0.0.90/go.mod h1:dcjd9P82mkBpFGEy0wBU/3qql5Bx14kmJZvVg7o7vXY= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.15 h1:jHHPo43s/TSiWmm6T8kJuMqTwL3ukU92iQhxq0K0jg0= -github.com/projectdiscovery/networkpolicy v0.1.15/go.mod h1:GWMDGJmgJ9qGoVTUOxbq1oLIbEx0pPsL0VKlriCkn2g= +github.com/projectdiscovery/networkpolicy v0.1.16 h1:H2VnLmMD7SvxF+rao+639nn8KX/kbPFY+mc8FxeltsI= +github.com/projectdiscovery/networkpolicy v0.1.16/go.mod h1:Vs/IRcJq4QUicjd/tl9gkhQWy7d/LssOwWbaz4buJ0U= github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= -github.com/projectdiscovery/retryabledns v1.0.101 h1:8DIVD8CL34Lc9h6KeOopPUfsPlcFxMlrnKOaI9VeOMk= -github.com/projectdiscovery/retryabledns v1.0.101/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= -github.com/projectdiscovery/retryablehttp-go v1.0.114 h1:JFvk7RJ2AUrHV9dScHcnyaBpQRGq1d8/QfrpccCT0xc= -github.com/projectdiscovery/retryablehttp-go v1.0.114/go.mod h1:ZXHlpbSw9w3nZqe1LH0GPX2UDAmv2QpUOoafy+xydYs= -github.com/projectdiscovery/utils v0.4.20 h1:7Fmjb+4YZJSzn7bL21sjF3wAR53eSi7VdAfDkDBUUwY= -github.com/projectdiscovery/utils v0.4.20/go.mod h1:RnC23+hI8j4drZFHQpMX92hV9++9d/yBeNr1pzcbF7Y= +github.com/projectdiscovery/retryabledns v1.0.102 h1:R8PzFCVofqLX3Bn4kdjOsE9wZ83FQjXZMDNs4/bHxzI= +github.com/projectdiscovery/retryabledns v1.0.102/go.mod h1:3+GL+YuHpV0Fp6UG7MbIG8mVxXHjfPO5ioQdwlnV08E= +github.com/projectdiscovery/retryablehttp-go v1.0.115 h1:ubIaVyHNj0/qxNv4gar+8/+L3G2Fhpfk54iMDctC7+E= +github.com/projectdiscovery/retryablehttp-go v1.0.115/go.mod h1:XlxLSMBVM7fTXeLVOLjVn1FLuRgQtD49NMFs9sQygfA= +github.com/projectdiscovery/utils v0.4.21 h1:yAothTUSF6NwZ9yoC4iGe5gSBrovqKR9JwwW3msxk3Q= +github.com/projectdiscovery/utils v0.4.21/go.mod h1:HJuJFqjB6EmVaDl0ilFPKvLoMaX2GyE6Il2TqKXNs8I= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= From 6867e076f1ceed4f0f63ec1cd8050886dcb66e20 Mon Sep 17 00:00:00 2001 From: Mikel Olasagasti Uranga Date: Thu, 10 Jul 2025 15:27:41 +0200 Subject: [PATCH 046/132] refactor: move v2 module code to root directory for cleaner structure Keeps module path as github.com/projectdiscovery/subfinder/v2 to preserve import compatibility. Signed-off-by: Mikel Olasagasti Uranga --- v2/.goreleaser.yml => .goreleaser.yml | 0 v2/Makefile => Makefile | 0 {v2/cmd => cmd}/subfinder/main.go | 0 {v2/examples => examples}/main.go | 0 v2/go.mod => go.mod | 0 v2/go.sum => go.sum | 0 {v2/pkg => pkg}/passive/doc.go | 0 {v2/pkg => pkg}/passive/passive.go | 0 {v2/pkg => pkg}/passive/sources.go | 0 {v2/pkg => pkg}/passive/sources_test.go | 0 {v2/pkg => pkg}/passive/sources_w_auth_test.go | 0 {v2/pkg => pkg}/passive/sources_wo_auth_test.go | 0 {v2/pkg => pkg}/resolve/client.go | 0 {v2/pkg => pkg}/resolve/doc.go | 0 {v2/pkg => pkg}/resolve/resolve.go | 0 {v2/pkg => pkg}/runner/banners.go | 0 {v2/pkg => pkg}/runner/config.go | 0 {v2/pkg => pkg}/runner/doc.go | 0 {v2/pkg => pkg}/runner/enumerate.go | 0 {v2/pkg => pkg}/runner/enumerate_test.go | 0 {v2/pkg => pkg}/runner/initialize.go | 0 {v2/pkg => pkg}/runner/options.go | 0 {v2/pkg => pkg}/runner/outputter.go | 0 {v2/pkg => pkg}/runner/runner.go | 0 {v2/pkg => pkg}/runner/stats.go | 0 {v2/pkg => pkg}/runner/util.go | 0 {v2/pkg => pkg}/runner/validate.go | 0 {v2/pkg => pkg}/subscraping/agent.go | 0 {v2/pkg => pkg}/subscraping/doc.go | 0 {v2/pkg => pkg}/subscraping/extractor.go | 0 {v2/pkg => pkg}/subscraping/sources/alienvault/alienvault.go | 0 {v2/pkg => pkg}/subscraping/sources/anubis/anubis.go | 0 {v2/pkg => pkg}/subscraping/sources/bevigil/bevigil.go | 0 {v2/pkg => pkg}/subscraping/sources/bufferover/bufferover.go | 0 {v2/pkg => pkg}/subscraping/sources/builtwith/builtwith.go | 0 {v2/pkg => pkg}/subscraping/sources/c99/c99.go | 0 {v2/pkg => pkg}/subscraping/sources/censys/censys.go | 0 {v2/pkg => pkg}/subscraping/sources/certspotter/certspotter.go | 0 {v2/pkg => pkg}/subscraping/sources/chaos/chaos.go | 0 {v2/pkg => pkg}/subscraping/sources/chinaz/chinaz.go | 0 {v2/pkg => pkg}/subscraping/sources/commoncrawl/commoncrawl.go | 0 {v2/pkg => pkg}/subscraping/sources/crtsh/crtsh.go | 0 {v2/pkg => pkg}/subscraping/sources/digitalyama/digitalyama.go | 0 {v2/pkg => pkg}/subscraping/sources/digitorus/digitorus.go | 0 {v2/pkg => pkg}/subscraping/sources/dnsdb/dnsdb.go | 0 {v2/pkg => pkg}/subscraping/sources/dnsdumpster/dnsdumpster.go | 0 {v2/pkg => pkg}/subscraping/sources/dnsrepo/dnsrepo.go | 0 {v2/pkg => pkg}/subscraping/sources/facebook/ctlogs.go | 0 {v2/pkg => pkg}/subscraping/sources/facebook/ctlogs_test.go | 0 {v2/pkg => pkg}/subscraping/sources/facebook/types.go | 0 {v2/pkg => pkg}/subscraping/sources/fofa/fofa.go | 0 {v2/pkg => pkg}/subscraping/sources/fullhunt/fullhunt.go | 0 {v2/pkg => pkg}/subscraping/sources/github/github.go | 0 {v2/pkg => pkg}/subscraping/sources/github/tokenmanager.go | 0 {v2/pkg => pkg}/subscraping/sources/gitlab/gitlab.go | 0 {v2/pkg => pkg}/subscraping/sources/hackertarget/hackertarget.go | 0 {v2/pkg => pkg}/subscraping/sources/hudsonrock/hudsonrock.go | 0 {v2/pkg => pkg}/subscraping/sources/hunter/hunter.go | 0 {v2/pkg => pkg}/subscraping/sources/intelx/intelx.go | 0 {v2/pkg => pkg}/subscraping/sources/leakix/leakix.go | 0 {v2/pkg => pkg}/subscraping/sources/netlas/netlas.go | 0 {v2/pkg => pkg}/subscraping/sources/pugrecon/pugrecon.go | 0 {v2/pkg => pkg}/subscraping/sources/quake/quake.go | 0 {v2/pkg => pkg}/subscraping/sources/rapiddns/rapiddns.go | 0 {v2/pkg => pkg}/subscraping/sources/reconcloud/reconcloud.go | 0 {v2/pkg => pkg}/subscraping/sources/redhuntlabs/redhuntlabs.go | 0 {v2/pkg => pkg}/subscraping/sources/riddler/riddler.go | 0 {v2/pkg => pkg}/subscraping/sources/robtex/robtext.go | 0 {v2/pkg => pkg}/subscraping/sources/rsecloud/rsecloud.go | 0 .../subscraping/sources/securitytrails/securitytrails.go | 0 {v2/pkg => pkg}/subscraping/sources/shodan/shodan.go | 0 {v2/pkg => pkg}/subscraping/sources/sitedossier/sitedossier.go | 0 {v2/pkg => pkg}/subscraping/sources/threatbook/threatbook.go | 0 {v2/pkg => pkg}/subscraping/sources/threatcrowd/threatcrowd.go | 0 {v2/pkg => pkg}/subscraping/sources/threatminer/threatminer.go | 0 {v2/pkg => pkg}/subscraping/sources/virustotal/virustotal.go | 0 .../subscraping/sources/waybackarchive/waybackarchive.go | 0 {v2/pkg => pkg}/subscraping/sources/whoisxmlapi/whoisxmlapi.go | 0 {v2/pkg => pkg}/subscraping/sources/zoomeyeapi/zoomeyeapi.go | 0 {v2/pkg => pkg}/subscraping/types.go | 0 {v2/pkg => pkg}/subscraping/utils.go | 0 {v2/pkg => pkg}/testutils/integration.go | 0 82 files changed, 0 insertions(+), 0 deletions(-) rename v2/.goreleaser.yml => .goreleaser.yml (100%) rename v2/Makefile => Makefile (100%) rename {v2/cmd => cmd}/subfinder/main.go (100%) rename {v2/examples => examples}/main.go (100%) rename v2/go.mod => go.mod (100%) rename v2/go.sum => go.sum (100%) rename {v2/pkg => pkg}/passive/doc.go (100%) rename {v2/pkg => pkg}/passive/passive.go (100%) rename {v2/pkg => pkg}/passive/sources.go (100%) rename {v2/pkg => pkg}/passive/sources_test.go (100%) rename {v2/pkg => pkg}/passive/sources_w_auth_test.go (100%) rename {v2/pkg => pkg}/passive/sources_wo_auth_test.go (100%) rename {v2/pkg => pkg}/resolve/client.go (100%) rename {v2/pkg => pkg}/resolve/doc.go (100%) rename {v2/pkg => pkg}/resolve/resolve.go (100%) rename {v2/pkg => pkg}/runner/banners.go (100%) rename {v2/pkg => pkg}/runner/config.go (100%) rename {v2/pkg => pkg}/runner/doc.go (100%) rename {v2/pkg => pkg}/runner/enumerate.go (100%) rename {v2/pkg => pkg}/runner/enumerate_test.go (100%) rename {v2/pkg => pkg}/runner/initialize.go (100%) rename {v2/pkg => pkg}/runner/options.go (100%) rename {v2/pkg => pkg}/runner/outputter.go (100%) rename {v2/pkg => pkg}/runner/runner.go (100%) rename {v2/pkg => pkg}/runner/stats.go (100%) rename {v2/pkg => pkg}/runner/util.go (100%) rename {v2/pkg => pkg}/runner/validate.go (100%) rename {v2/pkg => pkg}/subscraping/agent.go (100%) rename {v2/pkg => pkg}/subscraping/doc.go (100%) rename {v2/pkg => pkg}/subscraping/extractor.go (100%) rename {v2/pkg => pkg}/subscraping/sources/alienvault/alienvault.go (100%) rename {v2/pkg => pkg}/subscraping/sources/anubis/anubis.go (100%) rename {v2/pkg => pkg}/subscraping/sources/bevigil/bevigil.go (100%) rename {v2/pkg => pkg}/subscraping/sources/bufferover/bufferover.go (100%) rename {v2/pkg => pkg}/subscraping/sources/builtwith/builtwith.go (100%) rename {v2/pkg => pkg}/subscraping/sources/c99/c99.go (100%) rename {v2/pkg => pkg}/subscraping/sources/censys/censys.go (100%) rename {v2/pkg => pkg}/subscraping/sources/certspotter/certspotter.go (100%) rename {v2/pkg => pkg}/subscraping/sources/chaos/chaos.go (100%) rename {v2/pkg => pkg}/subscraping/sources/chinaz/chinaz.go (100%) rename {v2/pkg => pkg}/subscraping/sources/commoncrawl/commoncrawl.go (100%) rename {v2/pkg => pkg}/subscraping/sources/crtsh/crtsh.go (100%) rename {v2/pkg => pkg}/subscraping/sources/digitalyama/digitalyama.go (100%) rename {v2/pkg => pkg}/subscraping/sources/digitorus/digitorus.go (100%) rename {v2/pkg => pkg}/subscraping/sources/dnsdb/dnsdb.go (100%) rename {v2/pkg => pkg}/subscraping/sources/dnsdumpster/dnsdumpster.go (100%) rename {v2/pkg => pkg}/subscraping/sources/dnsrepo/dnsrepo.go (100%) rename {v2/pkg => pkg}/subscraping/sources/facebook/ctlogs.go (100%) rename {v2/pkg => pkg}/subscraping/sources/facebook/ctlogs_test.go (100%) rename {v2/pkg => pkg}/subscraping/sources/facebook/types.go (100%) rename {v2/pkg => pkg}/subscraping/sources/fofa/fofa.go (100%) rename {v2/pkg => pkg}/subscraping/sources/fullhunt/fullhunt.go (100%) rename {v2/pkg => pkg}/subscraping/sources/github/github.go (100%) rename {v2/pkg => pkg}/subscraping/sources/github/tokenmanager.go (100%) rename {v2/pkg => pkg}/subscraping/sources/gitlab/gitlab.go (100%) rename {v2/pkg => pkg}/subscraping/sources/hackertarget/hackertarget.go (100%) rename {v2/pkg => pkg}/subscraping/sources/hudsonrock/hudsonrock.go (100%) rename {v2/pkg => pkg}/subscraping/sources/hunter/hunter.go (100%) rename {v2/pkg => pkg}/subscraping/sources/intelx/intelx.go (100%) rename {v2/pkg => pkg}/subscraping/sources/leakix/leakix.go (100%) rename {v2/pkg => pkg}/subscraping/sources/netlas/netlas.go (100%) rename {v2/pkg => pkg}/subscraping/sources/pugrecon/pugrecon.go (100%) rename {v2/pkg => pkg}/subscraping/sources/quake/quake.go (100%) rename {v2/pkg => pkg}/subscraping/sources/rapiddns/rapiddns.go (100%) rename {v2/pkg => pkg}/subscraping/sources/reconcloud/reconcloud.go (100%) rename {v2/pkg => pkg}/subscraping/sources/redhuntlabs/redhuntlabs.go (100%) rename {v2/pkg => pkg}/subscraping/sources/riddler/riddler.go (100%) rename {v2/pkg => pkg}/subscraping/sources/robtex/robtext.go (100%) rename {v2/pkg => pkg}/subscraping/sources/rsecloud/rsecloud.go (100%) rename {v2/pkg => pkg}/subscraping/sources/securitytrails/securitytrails.go (100%) rename {v2/pkg => pkg}/subscraping/sources/shodan/shodan.go (100%) rename {v2/pkg => pkg}/subscraping/sources/sitedossier/sitedossier.go (100%) rename {v2/pkg => pkg}/subscraping/sources/threatbook/threatbook.go (100%) rename {v2/pkg => pkg}/subscraping/sources/threatcrowd/threatcrowd.go (100%) rename {v2/pkg => pkg}/subscraping/sources/threatminer/threatminer.go (100%) rename {v2/pkg => pkg}/subscraping/sources/virustotal/virustotal.go (100%) rename {v2/pkg => pkg}/subscraping/sources/waybackarchive/waybackarchive.go (100%) rename {v2/pkg => pkg}/subscraping/sources/whoisxmlapi/whoisxmlapi.go (100%) rename {v2/pkg => pkg}/subscraping/sources/zoomeyeapi/zoomeyeapi.go (100%) rename {v2/pkg => pkg}/subscraping/types.go (100%) rename {v2/pkg => pkg}/subscraping/utils.go (100%) rename {v2/pkg => pkg}/testutils/integration.go (100%) diff --git a/v2/.goreleaser.yml b/.goreleaser.yml similarity index 100% rename from v2/.goreleaser.yml rename to .goreleaser.yml diff --git a/v2/Makefile b/Makefile similarity index 100% rename from v2/Makefile rename to Makefile diff --git a/v2/cmd/subfinder/main.go b/cmd/subfinder/main.go similarity index 100% rename from v2/cmd/subfinder/main.go rename to cmd/subfinder/main.go diff --git a/v2/examples/main.go b/examples/main.go similarity index 100% rename from v2/examples/main.go rename to examples/main.go diff --git a/v2/go.mod b/go.mod similarity index 100% rename from v2/go.mod rename to go.mod diff --git a/v2/go.sum b/go.sum similarity index 100% rename from v2/go.sum rename to go.sum diff --git a/v2/pkg/passive/doc.go b/pkg/passive/doc.go similarity index 100% rename from v2/pkg/passive/doc.go rename to pkg/passive/doc.go diff --git a/v2/pkg/passive/passive.go b/pkg/passive/passive.go similarity index 100% rename from v2/pkg/passive/passive.go rename to pkg/passive/passive.go diff --git a/v2/pkg/passive/sources.go b/pkg/passive/sources.go similarity index 100% rename from v2/pkg/passive/sources.go rename to pkg/passive/sources.go diff --git a/v2/pkg/passive/sources_test.go b/pkg/passive/sources_test.go similarity index 100% rename from v2/pkg/passive/sources_test.go rename to pkg/passive/sources_test.go diff --git a/v2/pkg/passive/sources_w_auth_test.go b/pkg/passive/sources_w_auth_test.go similarity index 100% rename from v2/pkg/passive/sources_w_auth_test.go rename to pkg/passive/sources_w_auth_test.go diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go similarity index 100% rename from v2/pkg/passive/sources_wo_auth_test.go rename to pkg/passive/sources_wo_auth_test.go diff --git a/v2/pkg/resolve/client.go b/pkg/resolve/client.go similarity index 100% rename from v2/pkg/resolve/client.go rename to pkg/resolve/client.go diff --git a/v2/pkg/resolve/doc.go b/pkg/resolve/doc.go similarity index 100% rename from v2/pkg/resolve/doc.go rename to pkg/resolve/doc.go diff --git a/v2/pkg/resolve/resolve.go b/pkg/resolve/resolve.go similarity index 100% rename from v2/pkg/resolve/resolve.go rename to pkg/resolve/resolve.go diff --git a/v2/pkg/runner/banners.go b/pkg/runner/banners.go similarity index 100% rename from v2/pkg/runner/banners.go rename to pkg/runner/banners.go diff --git a/v2/pkg/runner/config.go b/pkg/runner/config.go similarity index 100% rename from v2/pkg/runner/config.go rename to pkg/runner/config.go diff --git a/v2/pkg/runner/doc.go b/pkg/runner/doc.go similarity index 100% rename from v2/pkg/runner/doc.go rename to pkg/runner/doc.go diff --git a/v2/pkg/runner/enumerate.go b/pkg/runner/enumerate.go similarity index 100% rename from v2/pkg/runner/enumerate.go rename to pkg/runner/enumerate.go diff --git a/v2/pkg/runner/enumerate_test.go b/pkg/runner/enumerate_test.go similarity index 100% rename from v2/pkg/runner/enumerate_test.go rename to pkg/runner/enumerate_test.go diff --git a/v2/pkg/runner/initialize.go b/pkg/runner/initialize.go similarity index 100% rename from v2/pkg/runner/initialize.go rename to pkg/runner/initialize.go diff --git a/v2/pkg/runner/options.go b/pkg/runner/options.go similarity index 100% rename from v2/pkg/runner/options.go rename to pkg/runner/options.go diff --git a/v2/pkg/runner/outputter.go b/pkg/runner/outputter.go similarity index 100% rename from v2/pkg/runner/outputter.go rename to pkg/runner/outputter.go diff --git a/v2/pkg/runner/runner.go b/pkg/runner/runner.go similarity index 100% rename from v2/pkg/runner/runner.go rename to pkg/runner/runner.go diff --git a/v2/pkg/runner/stats.go b/pkg/runner/stats.go similarity index 100% rename from v2/pkg/runner/stats.go rename to pkg/runner/stats.go diff --git a/v2/pkg/runner/util.go b/pkg/runner/util.go similarity index 100% rename from v2/pkg/runner/util.go rename to pkg/runner/util.go diff --git a/v2/pkg/runner/validate.go b/pkg/runner/validate.go similarity index 100% rename from v2/pkg/runner/validate.go rename to pkg/runner/validate.go diff --git a/v2/pkg/subscraping/agent.go b/pkg/subscraping/agent.go similarity index 100% rename from v2/pkg/subscraping/agent.go rename to pkg/subscraping/agent.go diff --git a/v2/pkg/subscraping/doc.go b/pkg/subscraping/doc.go similarity index 100% rename from v2/pkg/subscraping/doc.go rename to pkg/subscraping/doc.go diff --git a/v2/pkg/subscraping/extractor.go b/pkg/subscraping/extractor.go similarity index 100% rename from v2/pkg/subscraping/extractor.go rename to pkg/subscraping/extractor.go diff --git a/v2/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go similarity index 100% rename from v2/pkg/subscraping/sources/alienvault/alienvault.go rename to pkg/subscraping/sources/alienvault/alienvault.go diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go similarity index 100% rename from v2/pkg/subscraping/sources/anubis/anubis.go rename to pkg/subscraping/sources/anubis/anubis.go diff --git a/v2/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go similarity index 100% rename from v2/pkg/subscraping/sources/bevigil/bevigil.go rename to pkg/subscraping/sources/bevigil/bevigil.go diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go similarity index 100% rename from v2/pkg/subscraping/sources/bufferover/bufferover.go rename to pkg/subscraping/sources/bufferover/bufferover.go diff --git a/v2/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go similarity index 100% rename from v2/pkg/subscraping/sources/builtwith/builtwith.go rename to pkg/subscraping/sources/builtwith/builtwith.go diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go similarity index 100% rename from v2/pkg/subscraping/sources/c99/c99.go rename to pkg/subscraping/sources/c99/c99.go diff --git a/v2/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go similarity index 100% rename from v2/pkg/subscraping/sources/censys/censys.go rename to pkg/subscraping/sources/censys/censys.go diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go similarity index 100% rename from v2/pkg/subscraping/sources/certspotter/certspotter.go rename to pkg/subscraping/sources/certspotter/certspotter.go diff --git a/v2/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go similarity index 100% rename from v2/pkg/subscraping/sources/chaos/chaos.go rename to pkg/subscraping/sources/chaos/chaos.go diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go similarity index 100% rename from v2/pkg/subscraping/sources/chinaz/chinaz.go rename to pkg/subscraping/sources/chinaz/chinaz.go diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go similarity index 100% rename from v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go rename to pkg/subscraping/sources/commoncrawl/commoncrawl.go diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go similarity index 100% rename from v2/pkg/subscraping/sources/crtsh/crtsh.go rename to pkg/subscraping/sources/crtsh/crtsh.go diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go similarity index 100% rename from v2/pkg/subscraping/sources/digitalyama/digitalyama.go rename to pkg/subscraping/sources/digitalyama/digitalyama.go diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go similarity index 100% rename from v2/pkg/subscraping/sources/digitorus/digitorus.go rename to pkg/subscraping/sources/digitorus/digitorus.go diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go similarity index 100% rename from v2/pkg/subscraping/sources/dnsdb/dnsdb.go rename to pkg/subscraping/sources/dnsdb/dnsdb.go diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go similarity index 100% rename from v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go rename to pkg/subscraping/sources/dnsdumpster/dnsdumpster.go diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go similarity index 100% rename from v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go rename to pkg/subscraping/sources/dnsrepo/dnsrepo.go diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go similarity index 100% rename from v2/pkg/subscraping/sources/facebook/ctlogs.go rename to pkg/subscraping/sources/facebook/ctlogs.go diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go b/pkg/subscraping/sources/facebook/ctlogs_test.go similarity index 100% rename from v2/pkg/subscraping/sources/facebook/ctlogs_test.go rename to pkg/subscraping/sources/facebook/ctlogs_test.go diff --git a/v2/pkg/subscraping/sources/facebook/types.go b/pkg/subscraping/sources/facebook/types.go similarity index 100% rename from v2/pkg/subscraping/sources/facebook/types.go rename to pkg/subscraping/sources/facebook/types.go diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go similarity index 100% rename from v2/pkg/subscraping/sources/fofa/fofa.go rename to pkg/subscraping/sources/fofa/fofa.go diff --git a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go similarity index 100% rename from v2/pkg/subscraping/sources/fullhunt/fullhunt.go rename to pkg/subscraping/sources/fullhunt/fullhunt.go diff --git a/v2/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go similarity index 100% rename from v2/pkg/subscraping/sources/github/github.go rename to pkg/subscraping/sources/github/github.go diff --git a/v2/pkg/subscraping/sources/github/tokenmanager.go b/pkg/subscraping/sources/github/tokenmanager.go similarity index 100% rename from v2/pkg/subscraping/sources/github/tokenmanager.go rename to pkg/subscraping/sources/github/tokenmanager.go diff --git a/v2/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go similarity index 100% rename from v2/pkg/subscraping/sources/gitlab/gitlab.go rename to pkg/subscraping/sources/gitlab/gitlab.go diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go similarity index 100% rename from v2/pkg/subscraping/sources/hackertarget/hackertarget.go rename to pkg/subscraping/sources/hackertarget/hackertarget.go diff --git a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go similarity index 100% rename from v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go rename to pkg/subscraping/sources/hudsonrock/hudsonrock.go diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/pkg/subscraping/sources/hunter/hunter.go similarity index 100% rename from v2/pkg/subscraping/sources/hunter/hunter.go rename to pkg/subscraping/sources/hunter/hunter.go diff --git a/v2/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go similarity index 100% rename from v2/pkg/subscraping/sources/intelx/intelx.go rename to pkg/subscraping/sources/intelx/intelx.go diff --git a/v2/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go similarity index 100% rename from v2/pkg/subscraping/sources/leakix/leakix.go rename to pkg/subscraping/sources/leakix/leakix.go diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go similarity index 100% rename from v2/pkg/subscraping/sources/netlas/netlas.go rename to pkg/subscraping/sources/netlas/netlas.go diff --git a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go similarity index 100% rename from v2/pkg/subscraping/sources/pugrecon/pugrecon.go rename to pkg/subscraping/sources/pugrecon/pugrecon.go diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go similarity index 100% rename from v2/pkg/subscraping/sources/quake/quake.go rename to pkg/subscraping/sources/quake/quake.go diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go similarity index 100% rename from v2/pkg/subscraping/sources/rapiddns/rapiddns.go rename to pkg/subscraping/sources/rapiddns/rapiddns.go diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go similarity index 100% rename from v2/pkg/subscraping/sources/reconcloud/reconcloud.go rename to pkg/subscraping/sources/reconcloud/reconcloud.go diff --git a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go similarity index 100% rename from v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go rename to pkg/subscraping/sources/redhuntlabs/redhuntlabs.go diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go similarity index 100% rename from v2/pkg/subscraping/sources/riddler/riddler.go rename to pkg/subscraping/sources/riddler/riddler.go diff --git a/v2/pkg/subscraping/sources/robtex/robtext.go b/pkg/subscraping/sources/robtex/robtext.go similarity index 100% rename from v2/pkg/subscraping/sources/robtex/robtext.go rename to pkg/subscraping/sources/robtex/robtext.go diff --git a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go similarity index 100% rename from v2/pkg/subscraping/sources/rsecloud/rsecloud.go rename to pkg/subscraping/sources/rsecloud/rsecloud.go diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go similarity index 100% rename from v2/pkg/subscraping/sources/securitytrails/securitytrails.go rename to pkg/subscraping/sources/securitytrails/securitytrails.go diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go similarity index 100% rename from v2/pkg/subscraping/sources/shodan/shodan.go rename to pkg/subscraping/sources/shodan/shodan.go diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go similarity index 100% rename from v2/pkg/subscraping/sources/sitedossier/sitedossier.go rename to pkg/subscraping/sources/sitedossier/sitedossier.go diff --git a/v2/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go similarity index 100% rename from v2/pkg/subscraping/sources/threatbook/threatbook.go rename to pkg/subscraping/sources/threatbook/threatbook.go diff --git a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go similarity index 100% rename from v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go rename to pkg/subscraping/sources/threatcrowd/threatcrowd.go diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go similarity index 100% rename from v2/pkg/subscraping/sources/threatminer/threatminer.go rename to pkg/subscraping/sources/threatminer/threatminer.go diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go similarity index 100% rename from v2/pkg/subscraping/sources/virustotal/virustotal.go rename to pkg/subscraping/sources/virustotal/virustotal.go diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go similarity index 100% rename from v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go rename to pkg/subscraping/sources/waybackarchive/waybackarchive.go diff --git a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go similarity index 100% rename from v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go rename to pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go similarity index 100% rename from v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go rename to pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go diff --git a/v2/pkg/subscraping/types.go b/pkg/subscraping/types.go similarity index 100% rename from v2/pkg/subscraping/types.go rename to pkg/subscraping/types.go diff --git a/v2/pkg/subscraping/utils.go b/pkg/subscraping/utils.go similarity index 100% rename from v2/pkg/subscraping/utils.go rename to pkg/subscraping/utils.go diff --git a/v2/pkg/testutils/integration.go b/pkg/testutils/integration.go similarity index 100% rename from v2/pkg/testutils/integration.go rename to pkg/testutils/integration.go From 51f1cd9c0de504c0eaf5d94fe837dc522d9d166c Mon Sep 17 00:00:00 2001 From: Mikel Olasagasti Uranga Date: Thu, 10 Jul 2025 15:28:42 +0200 Subject: [PATCH 047/132] refactor: change Dockerfile paths Signed-off-by: Mikel Olasagasti Uranga --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7d2291080..025cab042 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM golang:1.24-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app -WORKDIR /app/v2 RUN go mod download RUN go build ./cmd/subfinder @@ -11,6 +10,6 @@ RUN go build ./cmd/subfinder FROM alpine:latest RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates -COPY --from=build-env /app/v2/subfinder /usr/local/bin/ +COPY --from=build-env /app/subfinder /usr/local/bin/ ENTRYPOINT ["subfinder"] From 0d9014e17742b6e419eb775806b82a9de2f507e9 Mon Sep 17 00:00:00 2001 From: Mikel Olasagasti Uranga Date: Thu, 10 Jul 2025 15:34:42 +0200 Subject: [PATCH 048/132] refactor: adapt workflows to new non-v2 paths --- .github/workflows/build-test.yml | 10 +++------- .github/workflows/compat-checks.yaml | 2 +- .github/workflows/dockerhub-push.yml | 2 +- .github/workflows/release-binary.yml | 3 +-- .github/workflows/release-test.yml | 1 - 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 54bac718b..80298e488 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -22,13 +22,12 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 with: - go-version-file: v2/go.mod + go-version-file: go.mod - name: Run golangci-lint uses: golangci/golangci-lint-action@v8 with: version: latest args: --timeout 5m - working-directory: v2/ build: name: Test Builds @@ -41,9 +40,8 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 with: - go-version-file: v2/go.mod + go-version-file: go.mod - run: go build ./... - working-directory: v2/ - name: Run tests env: @@ -75,12 +73,10 @@ jobs: with: timeout_seconds: 360 max_attempts: 3 - command: cd v2; go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} + command: go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} - name: Race Condition Tests run: go build -race ./... - working-directory: v2/ - name: Run Example run: go run . - working-directory: v2/examples diff --git a/.github/workflows/compat-checks.yaml b/.github/workflows/compat-checks.yaml index 0b28347be..8efb822d2 100644 --- a/.github/workflows/compat-checks.yaml +++ b/.github/workflows/compat-checks.yaml @@ -16,5 +16,5 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go/compat-checks@master with: - go-version-file: 'v2/go.mod' + go-version-file: 'go.mod' diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 8c328ce4b..4409dcc30 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -37,4 +37,4 @@ jobs: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} \ No newline at end of file + tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 487b3f5d3..ea68f5d0b 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -25,9 +25,8 @@ jobs: with: args: "release --clean" version: latest - workdir: v2/ env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index d5b099447..1c8fbbf8b 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -26,4 +26,3 @@ jobs: with: args: "release --clean --snapshot" version: latest - workdir: v2 \ No newline at end of file From 90f4c00d8c05fd58b4559e71f2ff07486a86e5d2 Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 2 Jul 2025 18:57:19 +0100 Subject: [PATCH 049/132] Adds the "driftnet" source. Updates the httpRequestWrapper() function to allow 204 responses. --- pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 3 + pkg/subscraping/agent.go | 2 +- .../subscraping/sources/driftnet/driftnet.go | 184 ++++++++++++++++++ 4 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/subscraping/sources/driftnet/driftnet.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 20013042f..e74d874be 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -26,6 +26,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" @@ -70,6 +71,7 @@ var AllSources = [...]subscraping.Source{ &dnsdb.Source{}, &dnsdumpster.Source{}, &dnsrepo.Source{}, + &driftnet.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index a63760296..4febbf9fa 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -26,6 +26,7 @@ var ( "dnsdumpster", "dnsdb", "dnsrepo", + "driftnet", "fofa", "fullhunt", "github", @@ -72,6 +73,7 @@ var ( "digitorus", "dnsdumpster", "dnsrepo", + "driftnet", "fofa", "fullhunt", "hackertarget", @@ -101,6 +103,7 @@ var ( "crtsh", "dnsdb", "digitorus", + "driftnet", "hackertarget", "securitytrails", "virustotal", diff --git a/pkg/subscraping/agent.go b/pkg/subscraping/agent.go index 4f2cab8c6..e4e1441aa 100644 --- a/pkg/subscraping/agent.go +++ b/pkg/subscraping/agent.go @@ -137,7 +137,7 @@ func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Respo return nil, err } - if response.StatusCode != http.StatusOK { + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNoContent { requestURL, _ := url.QueryUnescape(request.URL.String()) gologger.Debug().MsgFunc(func() string { diff --git a/v2/pkg/subscraping/sources/driftnet/driftnet.go b/v2/pkg/subscraping/sources/driftnet/driftnet.go new file mode 100644 index 000000000..a8f49a2be --- /dev/null +++ b/v2/pkg/subscraping/sources/driftnet/driftnet.go @@ -0,0 +1,184 @@ +// Package virustotal logic +package driftnet + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the base URL for the driftnet API + baseURL = "https://api.driftnet.io/v1/" + + // summaryLimit is the size of the summary limit that we send to the API + summaryLimit = 10000 +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// endpointConfig describes a driftnet endpoint that can used +type endpointConfig struct { + // The API endpoint to be touched + endpoint string + + // The API parameter used for query + param string + + // The context that we should restrict to in results from this endpoint + context string +} + +// endpoints is a set of endpoint configs +var endpoints = []endpointConfig{ + {"ct/log", "field=host:", "cert-dns-name"}, + {"scan/protocols", "field=host:", "cert-dns-name"}, + {"scan/domains", "field=host:", "cert-dns-name"}, + {"domain/rdns", "host=", "dns-ptr"}, +} + +// summaryResponse is an API response +type summaryResponse struct { + Summary struct { + Other int `json:"other"` + Values map[string]int `json:"values"` + } `json:"summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + // Final results channel + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + // Waitgroup for subsources + var wg sync.WaitGroup + wg.Add(len(endpoints)) + + // Map for dedupe between subsources + dedupe := sync.Map{} + + // Close down results when all subsources finished + go func(startTime time.Time) { + wg.Wait() + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + // Start up requests for all subsources + for i := range endpoints { + go s.runSubsource(ctx, domain, session, results, &wg, &dedupe, endpoints[i]) + } + + // Return the result c + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "driftnet" +} + +// IsDefault indicates that this source should used as part of the default execution. +func (s *Source) IsDefault() bool { + return true +} + +// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains +func (s *Source) HasRecursiveSupport() bool { + return true +} + +// NeedsKey indicates that we need an API key +func (s *Source) NeedsKey() bool { + return true +} + +// AddApiKeys provides us with the API key(s) +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns statistics about the scraping process +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} + +// runSubsource +func (s *Source) runSubsource(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result, wg *sync.WaitGroup, dedupe *sync.Map, epConfig endpointConfig) { + // Default headers + headers := map[string]string{ + "accept": "application/json", + } + + // Pick an API key + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["authorization"] = "Bearer " + randomApiKey + } + + // Request + url := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, domain, epConfig.context, summaryLimit) + resp, err := session.Get(ctx, url, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + wg.Done() + return + } + + defer session.DiscardHTTPResponse(resp) + + // 204 means no results, any other response code is an error + if resp.StatusCode != 200 { + if resp.StatusCode != 204 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} + s.errors++ + } + + wg.Done() + return + } + + // Parse and return results + var summary summaryResponse + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&summary) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + wg.Done() + return + } + + for subdomain := range summary.Summary.Values { + // Avoid returning the same result more than once from the same source (can happen as we are using multiple endpoints) + if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain, + } + s.results++ + } + } + + // Complete! + wg.Done() +} From 0afa0e5a72c680c6828555dad128d764321ae02f Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 2 Jul 2025 19:33:28 +0100 Subject: [PATCH 050/132] remove cruft --- v2/pkg/subscraping/sources/driftnet/driftnet.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/driftnet/driftnet.go b/v2/pkg/subscraping/sources/driftnet/driftnet.go index a8f49a2be..8ff59f2df 100644 --- a/v2/pkg/subscraping/sources/driftnet/driftnet.go +++ b/v2/pkg/subscraping/sources/driftnet/driftnet.go @@ -1,4 +1,3 @@ -// Package virustotal logic package driftnet import ( From 8f5680e90690ceac4bb31b9a628259031a8899ea Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 17 Jul 2025 12:31:42 +0100 Subject: [PATCH 051/132] match new path --- {v2/pkg => pkg}/subscraping/sources/driftnet/driftnet.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {v2/pkg => pkg}/subscraping/sources/driftnet/driftnet.go (100%) diff --git a/v2/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go similarity index 100% rename from v2/pkg/subscraping/sources/driftnet/driftnet.go rename to pkg/subscraping/sources/driftnet/driftnet.go From 423ae72f084e0003a9e42f2ca66faf3aa0101545 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 17 Jul 2025 12:50:56 +0100 Subject: [PATCH 052/132] correct typos and fix a keys-required test --- pkg/subscraping/sources/driftnet/driftnet.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index 8ff59f2df..fc599ab4f 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -82,7 +82,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se go s.runSubsource(ctx, domain, session, results, &wg, &dedupe, endpoints[i]) } - // Return the result c + // Return the results channel return results } @@ -121,7 +121,7 @@ func (s *Source) Statistics() subscraping.Statistics { } } -// runSubsource +// runSubsource queries a specific driftnet endpoint for subdomains and sends results to the channel func (s *Source) runSubsource(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result, wg *sync.WaitGroup, dedupe *sync.Map, epConfig endpointConfig) { // Default headers headers := map[string]string{ @@ -169,6 +169,11 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc } for subdomain := range summary.Summary.Values { + // We can get certificate results which aren't actually subdomains of the target domain. Skip them. + if !strings.HasSuffix(subdomain, "."+domain) { + continue + } + // Avoid returning the same result more than once from the same source (can happen as we are using multiple endpoints) if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { results <- subscraping.Result{ From ed66a7d04e338e98eb7c36961330b3dbf7f6bda9 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 17 Jul 2025 12:51:22 +0100 Subject: [PATCH 053/132] revert change for NoContent --- pkg/subscraping/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/subscraping/agent.go b/pkg/subscraping/agent.go index e4e1441aa..4f2cab8c6 100644 --- a/pkg/subscraping/agent.go +++ b/pkg/subscraping/agent.go @@ -137,7 +137,7 @@ func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Respo return nil, err } - if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNoContent { + if response.StatusCode != http.StatusOK { requestURL, _ := url.QueryUnescape(request.URL.String()) gologger.Debug().MsgFunc(func() string { From e10fc7f9d75a407a84621ecb0772eb61499d8caa Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 17 Jul 2025 12:57:29 +0100 Subject: [PATCH 054/132] fix for race --- pkg/subscraping/sources/driftnet/driftnet.go | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index fc599ab4f..5e88fa8e0 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -23,8 +24,8 @@ const ( type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int + errors atomic.Int32 + results atomic.Int32 skipped bool } @@ -60,8 +61,8 @@ type summaryResponse struct { func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { // Final results channel results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 + s.errors.Store(0) + s.results.Store(0) // Waitgroup for subsources var wg sync.WaitGroup @@ -114,8 +115,8 @@ func (s *Source) AddApiKeys(keys []string) { // Statistics returns statistics about the scraping process func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } @@ -139,7 +140,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc resp, err := session.Get(ctx, url, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) wg.Done() return } @@ -150,7 +151,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc if resp.StatusCode != 200 { if resp.StatusCode != 204 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} - s.errors++ + s.errors.Add(1) } wg.Done() @@ -163,7 +164,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc err = decoder.Decode(&summary) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) wg.Done() return } @@ -179,7 +180,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain, } - s.results++ + s.results.Add(1) } } From b2f1963756c60d07842ee79743c0d57779c6b183 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 17 Jul 2025 12:58:45 +0100 Subject: [PATCH 055/132] url escape as requested --- pkg/subscraping/sources/driftnet/driftnet.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index 5e88fa8e0..37c1f8c5f 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net/url" "strings" "sync" "sync/atomic" @@ -136,8 +137,8 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc } // Request - url := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, domain, epConfig.context, summaryLimit) - resp, err := session.Get(ctx, url, "", headers) + requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) + resp, err := session.Get(ctx, requestURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) From dcc7f4b3cd097650b970bbc3e3a7377fd96e8d5d Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 17 Jul 2025 13:03:58 +0100 Subject: [PATCH 056/132] move 204 check to be consistent with other sources --- pkg/subscraping/sources/driftnet/driftnet.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index 37c1f8c5f..758bbfa44 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "net/url" "strings" "sync" @@ -140,8 +141,12 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) resp, err := session.Get(ctx, requestURL, "", headers) if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors.Add(1) + // HTTP 204 is not an error from the Driftnet API + if resp == nil || resp.StatusCode != http.StatusNoContent { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors.Add(1) + } + wg.Done() return } From a071cc2994b55a5427461760d5726e8913f1557c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:55:09 +0000 Subject: [PATCH 057/132] chore(deps): bump golang.org/x/oauth2 from 0.11.0 to 0.27.0 Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.11.0 to 0.27.0. - [Commits](https://github.com/golang/oauth2/compare/v0.11.0...v0.27.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-version: 0.27.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 5 +---- go.sum | 9 ++------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 273a3a328..436e17e84 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( github.com/fatih/color v1.15.0 // indirect github.com/gaissmai/bart v0.20.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -114,14 +113,12 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.29.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.33.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 80900dbb4..2aeebb583 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,6 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -483,8 +481,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -602,7 +600,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -628,8 +625,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From f45039b014bed97227864d539950577fdceb3c75 Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Mon, 21 Jul 2025 06:03:04 -0700 Subject: [PATCH 058/132] docs: update main.go reference Signed-off-by: Emmanuel Ferdman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a3ce144e..399998af8 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Learn about how to run Subfinder here: https://docs.projectdiscovery.io/tools/su ## Subfinder Go library -Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](v2/examples/main.go) +Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](examples/main.go) From e3af027548e2ec82386bbe9743b3c702fdebe4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 23 Jul 2025 19:06:56 +0300 Subject: [PATCH 059/132] add api key support for hackertarget --- pkg/runner/config.go | 2 +- .../sources/hackertarget/hackertarget.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/runner/config.go b/pkg/runner/config.go index 1751c9054..fe6a3da40 100644 --- a/pkg/runner/config.go +++ b/pkg/runner/config.go @@ -46,7 +46,7 @@ func UnmarshalFrom(file string) error { for _, source := range passive.AllSources { sourceName := strings.ToLower(source.Name()) apiKeys := sourceApiKeysMap[sourceName] - if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 { + if len(apiKeys) > 0 { gologger.Debug().Msgf("API key(s) found for %s.", sourceName) source.AddApiKeys(apiKeys) } diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index c84957722..cf5876c2c 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -12,6 +12,7 @@ import ( // Source is the passive scraping agent type Source struct { + apiKeys []string timeTaken time.Duration errors int results int @@ -29,7 +30,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain)) + htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) + if len(s.apiKeys) > 0 { + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, s.apiKeys[0]) + } + fmt.Println(htSearchUrl) + resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -78,8 +84,9 @@ func (s *Source) NeedsKey() bool { return false } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +// TODO: env variable will not work if NeedsKey is false, entire api key management needs to be refactored +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { From f786099a2e1979b2587acf76921c8db5a1ed9ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 23 Jul 2025 19:08:24 +0300 Subject: [PATCH 060/132] remove debug code --- pkg/subscraping/sources/hackertarget/hackertarget.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index cf5876c2c..d562f3783 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + skipped bool } // Run function returns all subdomains found with the service @@ -31,10 +32,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se }(time.Now()) htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) - if len(s.apiKeys) > 0 { - htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, s.apiKeys[0]) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return } - fmt.Println(htSearchUrl) + + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} From 31d55080dca032d802669b80a560ab2f35215b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 23 Jul 2025 19:08:43 +0300 Subject: [PATCH 061/132] minor --- pkg/subscraping/sources/hackertarget/hackertarget.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index d562f3783..cac388261 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -99,5 +99,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Skipped: s.skipped, } } From ae80ddae618095b272459636ae896ed2cda93e0a Mon Sep 17 00:00:00 2001 From: Douglas Danger Manley Date: Wed, 13 Aug 2025 09:13:33 -0400 Subject: [PATCH 062/132] Prevent a netlas nil pointer dereference This uses two _different_ variables for the two _different_ HTTP requests, since two `defer` functions referenced the same `resp` variable. This setup could cause a nil pointer dereference in the following scenario: 1. The first request succeeds (a `defer` on `resp` is added). 2. The second request fails (which sets `resp` to nil, and then the first `defer` attempts to reference `resp.Body`). This change prevents that by not reusing the same variable for the second request. --- pkg/subscraping/sources/netlas/netlas.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go index c29bc2c8b..cd9f612e6 100644 --- a/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -63,7 +63,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ + resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, }, nil, subscraping.BasicAuth{}) @@ -72,19 +72,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return - } else if resp.StatusCode != 200 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + } else if resp1.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp1.StatusCode)} s.errors++ return } defer func() { - if err := resp.Body.Close(); err != nil { + if err := resp1.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp1.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ @@ -120,7 +120,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ + resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) @@ -130,20 +130,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } defer func() { - if err := resp.Body.Close(); err != nil { + if err := resp2.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() - body, err = io.ReadAll(resp.Body) + body, err = io.ReadAll(resp2.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ return } - if resp.StatusCode == 429 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + if resp2.StatusCode == 429 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp2.StatusCode)} s.errors++ return } From f0ec4032b2cdc5b03867ea7a4f7b6a39b3cf40cb Mon Sep 17 00:00:00 2001 From: Douglas Danger Manley Date: Fri, 15 Aug 2025 10:13:22 -0400 Subject: [PATCH 063/132] Fix the `build-test` action (#1631) This fixes the `build-test` action by restoring the `examples` working directory. This was incorrectly removed in the previous commit. --- .github/workflows/build-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 80298e488..7a5080adf 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -80,3 +80,4 @@ jobs: - name: Run Example run: go run . + working-directory: examples From 5d50a55a122623faede3240c638c648d22652852 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Aug 2025 19:43:21 +0000 Subject: [PATCH 064/132] chore(deps): bump github.com/ulikunitz/xz from 0.5.12 to 0.5.14 Bumps [github.com/ulikunitz/xz](https://github.com/ulikunitz/xz) from 0.5.12 to 0.5.14. - [Commits](https://github.com/ulikunitz/xz/compare/v0.5.12...v0.5.14) --- updated-dependencies: - dependency-name: github.com/ulikunitz/xz dependency-version: 0.5.14 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 436e17e84..bc2182b42 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,7 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect + github.com/ulikunitz/xz v0.5.14 // indirect github.com/weppos/publicsuffix-go v0.30.1 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect diff --git a/go.sum b/go.sum index 2aeebb583..38f57585e 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= -github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg= +github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= From d29e24f9134cdb62746980c1b2b859c41bc26151 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 12 Sep 2025 16:39:43 +0200 Subject: [PATCH 065/132] bump --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bc2182b42..e09106058 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,7 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.14 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect github.com/weppos/publicsuffix-go v0.30.1 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect diff --git a/go.sum b/go.sum index 38f57585e..e52607b63 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg= -github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= From 92ee44cb13570858bf2316511c62f210d7943fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 16 Sep 2025 14:27:54 +0300 Subject: [PATCH 066/132] bump version --- pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index a0d3a7d1c..d02334782 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.8.0` +const version = `v2.9.0` // showBanner is used to show the banner to the user func showBanner() { From e5bcc681d15d5088f7ba0bd59eabff12d05c2f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 22 Sep 2025 16:00:01 +0300 Subject: [PATCH 067/132] add onhype --- pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 2 + pkg/subscraping/sources/onhype/onhype.go | 129 +++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 pkg/subscraping/sources/onhype/onhype.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index e74d874be..f91fdf860 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -37,6 +37,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onhype" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -79,6 +80,7 @@ var AllSources = [...]subscraping.Source{ &hunter.Source{}, &intelx.Source{}, &netlas.Source{}, + &onhype.Source{}, &leakix.Source{}, &quake.Source{}, &pugrecon.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 4febbf9fa..53f8a6b92 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -33,6 +33,7 @@ var ( "hackertarget", "intelx", "netlas", + "onhype", "quake", "pugrecon", "rapiddns", @@ -78,6 +79,7 @@ var ( "fullhunt", "hackertarget", "intelx", + "onhype", "quake", "redhuntlabs", "robtex", diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onhype/onhype.go new file mode 100644 index 000000000..916ed51b5 --- /dev/null +++ b/pkg/subscraping/sources/onhype/onhype.go @@ -0,0 +1,129 @@ +// Package onhype logic +package onhype + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type OnypheResponse struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Total int `json:"total"` +} + +type Result struct { + Hostname string `json:"hostname"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} + + page := 1 + for { + var resp *http.Response + var err error + + urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=10", + url.QueryEscape("category:resolver domain:"+domain), page) + resp, err = session.Get(ctx, urlWithQuery, "", headers) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var respOnyphe OnypheResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&respOnyphe) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + session.DiscardHTTPResponse(resp) + + for _, record := range respOnyphe.Results { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } + + if len(respOnyphe.Results) == 0 || (respOnyphe.Page)*respOnyphe.PageSize >= respOnyphe.Total { + break + } + + page++ + + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "onhype" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 0ed6faf8f26735de44f5b9183d195aefc9c56b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 24 Sep 2025 13:46:52 +0300 Subject: [PATCH 068/132] add env var support for config files --- pkg/runner/options.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/runner/options.go b/pkg/runner/options.go index 7f39e0315..aee357811 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -118,6 +118,14 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.HostIP, "ip", "oI", false, "include host IP in output (-active only)"), ) + if envConfig := os.Getenv("SUBFINDER_CONFIG"); envConfig != "" { + defaultConfigLocation = envConfig + } + + if envProviderConfig := os.Getenv("SUBFINDER_PROVIDER_CONFIG"); envProviderConfig != "" { + defaultProviderConfigLocation = envProviderConfig + } + flagSet.CreateGroup("configuration", "Configuration", flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "flag config file"), flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider config file"), From 20682b595ab354699f71bdc673ebf3a9c7fd99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 24 Sep 2025 13:52:17 +0300 Subject: [PATCH 069/132] update README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 399998af8..dad545074 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,13 @@ OPTIMIZATION: -max-time int minutes to wait for enumeration results (default 10) ``` +## Environment Variables + +Subfinder supports environment variables to specify custom paths for configuration files: + +- `SUBFINDER_CONFIG` - Path to config.yaml file (overrides default `$CONFIG/subfinder/config.yaml`) +- `SUBFINDER_PROVIDER_CONFIG` - Path to provider-config.yaml file (overrides default `$CONFIG/subfinder/provider-config.yaml`) + # Installation `subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: From e9338c217b8c284ba39922e3247b4a6d1f0eb1c5 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Sep 2025 20:19:34 +0200 Subject: [PATCH 070/132] using envutil --- pkg/runner/options.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/runner/options.go b/pkg/runner/options.go index aee357811..a2fd4d411 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + envutil "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" @@ -23,8 +24,8 @@ import ( var ( configDir = folderutil.AppConfigDirOrDefault(".", "subfinder") - defaultConfigLocation = filepath.Join(configDir, "config.yaml") - defaultProviderConfigLocation = filepath.Join(configDir, "provider-config.yaml") + defaultConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_CONFIG", filepath.Join(configDir, "config.yaml")) + defaultProviderConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_PROVIDER_CONFIG", filepath.Join(configDir, "provider-config.yaml")) ) // Options contains the configuration options for tuning @@ -118,14 +119,6 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.HostIP, "ip", "oI", false, "include host IP in output (-active only)"), ) - if envConfig := os.Getenv("SUBFINDER_CONFIG"); envConfig != "" { - defaultConfigLocation = envConfig - } - - if envProviderConfig := os.Getenv("SUBFINDER_PROVIDER_CONFIG"); envProviderConfig != "" { - defaultProviderConfigLocation = envProviderConfig - } - flagSet.CreateGroup("configuration", "Configuration", flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "flag config file"), flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider config file"), From 68a34884ab033555ce25168c0efc48d1bfd270e7 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Sep 2025 20:23:39 +0200 Subject: [PATCH 071/132] latest osx --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7a5080adf..e6fabe953 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-13] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 From c292a9c595506e9ae47543d2c501eeeb6f5d50ea Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Sep 2025 20:44:14 +0200 Subject: [PATCH 072/132] ignoring flaky threatcrowd --- pkg/passive/sources_wo_auth_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go index 036588cc3..01fc57d98 100644 --- a/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -33,6 +33,7 @@ func TestSourcesWithoutKeys(t *testing.T) { "digitorus", // failing with "Failed to retrieve certificate" "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" + "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" } domain := "hackerone.com" From 2097be5c7075a8c89c8c522d6117197e1ca97197 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Wed, 1 Oct 2025 21:07:10 +0530 Subject: [PATCH 073/132] Call ResultCallback sooner --- pkg/runner/enumerate.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index f71a347dd..99be92732 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -94,6 +94,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string } hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source} + if r.options.ResultCallback != nil { + r.options.ResultCallback(&hostEntry) + } uniqueMap[subdomain] = hostEntry // If the user asked to remove wildcard then send on the resolve @@ -162,17 +165,6 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string numberOfSubDomains = len(uniqueMap) } - if r.options.ResultCallback != nil { - if r.options.RemoveWildcard { - for host, result := range foundResults { - r.options.ResultCallback(&resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source}) - } - } else { - for _, v := range uniqueMap { - r.options.ResultCallback(&v) - } - } - } gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration) if r.options.Statistics { From 7e6ad511810df38c1ea80c61a474a6cee488cfa1 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Wed, 1 Oct 2025 21:32:23 +0530 Subject: [PATCH 074/132] Handle case when RemoveWildcard is true --- pkg/runner/enumerate.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index 99be92732..736f9d452 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -94,7 +94,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string } hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source} - if r.options.ResultCallback != nil { + if r.options.ResultCallback != nil && !r.options.RemoveWildcard { r.options.ResultCallback(&hostEntry) } @@ -128,6 +128,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // Add the found subdomain to a map. if _, ok := foundResults[result.Host]; !ok { foundResults[result.Host] = result + if r.options.ResultCallback != nil { + r.options.ResultCallback(&resolve.HostEntry{Host: result.Host, Source: result.Source}) + } } } } From 50bbe41385b958f7e2d979a7f91ab1a85a866033 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Wed, 1 Oct 2025 22:23:24 +0530 Subject: [PATCH 075/132] Add missing domain field --- pkg/runner/enumerate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index 736f9d452..f39eab068 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -129,7 +129,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string if _, ok := foundResults[result.Host]; !ok { foundResults[result.Host] = result if r.options.ResultCallback != nil { - r.options.ResultCallback(&resolve.HostEntry{Host: result.Host, Source: result.Source}) + r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source}) } } } From 648f1e131125db26632d0a08d84dbd29a11e6957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 2 Oct 2025 13:27:22 +0300 Subject: [PATCH 076/132] fix pagination --- pkg/subscraping/sources/onhype/onhype.go | 191 +++++++++++++++++++++-- 1 file changed, 181 insertions(+), 10 deletions(-) diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onhype/onhype.go index 916ed51b5..6eb33d68a 100644 --- a/pkg/subscraping/sources/onhype/onhype.go +++ b/pkg/subscraping/sources/onhype/onhype.go @@ -3,13 +3,13 @@ package onhype import ( "context" + "encoding/json" "fmt" "net/http" "net/url" + "strconv" "time" - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) @@ -19,13 +19,18 @@ type OnypheResponse struct { Page int `json:"page"` PageSize int `json:"page_size"` Total int `json:"total"` + MaxPage int `json:"max_page"` } type Result struct { - Hostname string `json:"hostname"` + Subdomains []string `json:"subdomains"` + Hostname string `json:"hostname"` + Forward string `json:"forward"` + Reverse string `json:"reverse"` + Host string `json:"host"` + Domain string `json:"domain"` } -// Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration @@ -55,12 +60,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} page := 1 + pageSize := 1000 + for { var resp *http.Response var err error - urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=10", - url.QueryEscape("category:resolver domain:"+domain), page) + urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", + url.QueryEscape("category:resolver domain:"+domain), page, pageSize) resp, err = session.Get(ctx, urlWithQuery, "", headers) if err != nil { @@ -71,7 +78,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var respOnyphe OnypheResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&respOnyphe) + err = json.NewDecoder(resp.Body).Decode(&respOnyphe) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -82,11 +89,30 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range respOnyphe.Results { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + for _, subdomain := range record.Subdomains { + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + + if record.Hostname != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } + + if record.Forward != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Forward} + s.results++ + } + + if record.Reverse != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Reverse} + s.results++ + } } - if len(respOnyphe.Results) == 0 || (respOnyphe.Page)*respOnyphe.PageSize >= respOnyphe.Total { + if len(respOnyphe.Results) == 0 || page >= respOnyphe.MaxPage { break } @@ -127,3 +153,148 @@ func (s *Source) Statistics() subscraping.Statistics { Skipped: s.skipped, } } + +type OnypheResponseRaw struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page json.RawMessage `json:"page"` + PageSize json.RawMessage `json:"page_size"` + Total json.RawMessage `json:"total"` + MaxPage json.RawMessage `json:"max_page"` +} + +func (o *OnypheResponse) UnmarshalJSON(data []byte) error { + var raw OnypheResponseRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + o.Error = raw.Error + o.Results = raw.Results + + if pageStr := string(raw.Page); pageStr != "" { + if page, err := strconv.Atoi(pageStr); err == nil { + o.Page = page + } else { + var pageStrQuoted string + if err := json.Unmarshal(raw.Page, &pageStrQuoted); err == nil { + if page, err := strconv.Atoi(pageStrQuoted); err == nil { + o.Page = page + } + } + } + } + + if pageSizeStr := string(raw.PageSize); pageSizeStr != "" { + if pageSize, err := strconv.Atoi(pageSizeStr); err == nil { + o.PageSize = pageSize + } else { + var pageSizeStrQuoted string + if err := json.Unmarshal(raw.PageSize, &pageSizeStrQuoted); err == nil { + if pageSize, err := strconv.Atoi(pageSizeStrQuoted); err == nil { + o.PageSize = pageSize + } + } + } + } + + if totalStr := string(raw.Total); totalStr != "" { + if total, err := strconv.Atoi(totalStr); err == nil { + o.Total = total + } else { + var totalStrQuoted string + if err := json.Unmarshal(raw.Total, &totalStrQuoted); err == nil { + if total, err := strconv.Atoi(totalStrQuoted); err == nil { + o.Total = total + } + } + } + } + + if maxPageStr := string(raw.MaxPage); maxPageStr != "" { + if maxPage, err := strconv.Atoi(maxPageStr); err == nil { + o.MaxPage = maxPage + } else { + var maxPageStrQuoted string + if err := json.Unmarshal(raw.MaxPage, &maxPageStrQuoted); err == nil { + if maxPage, err := strconv.Atoi(maxPageStrQuoted); err == nil { + o.MaxPage = maxPage + } + } + } + } + + return nil +} + +type ResultRaw struct { + Subdomains json.RawMessage `json:"subdomains"` + Hostname json.RawMessage `json:"hostname"` + Forward json.RawMessage `json:"forward"` + Reverse json.RawMessage `json:"reverse"` + Host json.RawMessage `json:"host"` + Domain json.RawMessage `json:"domain"` +} + +func (r *Result) UnmarshalJSON(data []byte) error { + var raw ResultRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + var subdomains []string + if err := json.Unmarshal(raw.Subdomains, &subdomains); err == nil { + r.Subdomains = subdomains + } else { + var subdomainStr string + if err := json.Unmarshal(raw.Subdomains, &subdomainStr); err == nil { + r.Subdomains = []string{subdomainStr} + } + } + + if len(raw.Hostname) > 0 { + var hostnameStr string + if err := json.Unmarshal(raw.Hostname, &hostnameStr); err == nil { + r.Hostname = hostnameStr + } else { + var hostnameArr []string + if err := json.Unmarshal(raw.Hostname, &hostnameArr); err == nil && len(hostnameArr) > 0 { + r.Hostname = hostnameArr[0] + } + } + } + + if len(raw.Forward) > 0 { + json.Unmarshal(raw.Forward, &r.Forward) + } + + if len(raw.Reverse) > 0 { + json.Unmarshal(raw.Reverse, &r.Reverse) + } + + if len(raw.Host) > 0 { + var hostStr string + if err := json.Unmarshal(raw.Host, &hostStr); err == nil { + r.Host = hostStr + } else { + var hostArr []string + if err := json.Unmarshal(raw.Host, &hostArr); err == nil && len(hostArr) > 0 { + r.Host = hostArr[0] + } + } + } + + if len(raw.Domain) > 0 { + var domainStr string + if err := json.Unmarshal(raw.Domain, &domainStr); err == nil { + r.Domain = domainStr + } else { + var domainArr []string + if err := json.Unmarshal(raw.Domain, &domainArr); err == nil && len(domainArr) > 0 { + r.Domain = domainArr[0] + } + } + } + + return nil +} From d236bba3dd2151cb57ebc36f64a0e9665dff1da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 2 Oct 2025 13:32:23 +0300 Subject: [PATCH 077/132] fix lint err --- pkg/subscraping/sources/onhype/onhype.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onhype/onhype.go index 6eb33d68a..6a571081a 100644 --- a/pkg/subscraping/sources/onhype/onhype.go +++ b/pkg/subscraping/sources/onhype/onhype.go @@ -265,11 +265,11 @@ func (r *Result) UnmarshalJSON(data []byte) error { } if len(raw.Forward) > 0 { - json.Unmarshal(raw.Forward, &r.Forward) + _ = json.Unmarshal(raw.Forward, &r.Forward) } if len(raw.Reverse) > 0 { - json.Unmarshal(raw.Reverse, &r.Reverse) + _ = json.Unmarshal(raw.Reverse, &r.Reverse) } if len(raw.Host) > 0 { From 7823d41dd12ed53bf1fee4b0a41af3a25b360e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 2 Oct 2025 13:35:06 +0300 Subject: [PATCH 078/132] fix typo --- pkg/passive/sources.go | 4 ++-- pkg/passive/sources_test.go | 4 ++-- .../sources/{onhype/onhype.go => onyphe/onyphe.go} | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename pkg/subscraping/sources/{onhype/onhype.go => onyphe/onyphe.go} (99%) diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index f91fdf860..a2065b415 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -37,7 +37,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onhype" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -80,7 +80,7 @@ var AllSources = [...]subscraping.Source{ &hunter.Source{}, &intelx.Source{}, &netlas.Source{}, - &onhype.Source{}, + &onyphe.Source{}, &leakix.Source{}, &quake.Source{}, &pugrecon.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 53f8a6b92..540990ecd 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -33,7 +33,7 @@ var ( "hackertarget", "intelx", "netlas", - "onhype", + "onyphe", "quake", "pugrecon", "rapiddns", @@ -79,7 +79,7 @@ var ( "fullhunt", "hackertarget", "intelx", - "onhype", + "onyphe", "quake", "redhuntlabs", "robtex", diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onyphe/onyphe.go similarity index 99% rename from pkg/subscraping/sources/onhype/onhype.go rename to pkg/subscraping/sources/onyphe/onyphe.go index 6a571081a..6ea1dcbb1 100644 --- a/pkg/subscraping/sources/onhype/onhype.go +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -1,5 +1,5 @@ -// Package onhype logic -package onhype +// Package onyphe logic +package onyphe import ( "context" @@ -126,7 +126,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Name returns the name of the source func (s *Source) Name() string { - return "onhype" + return "onyphe" } func (s *Source) IsDefault() bool { From c47f852d470363d1520d5d8454a4d8cd84c80ead Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Oct 2025 00:34:43 +0000 Subject: [PATCH 079/132] chore(deps): bump github.com/nwaples/rardecode/v2 Bumps [github.com/nwaples/rardecode/v2](https://github.com/nwaples/rardecode) from 2.0.0-beta.4.0.20241112120701-034e449c6e78 to 2.2.0. - [Commits](https://github.com/nwaples/rardecode/commits/v2.2.0) --- updated-dependencies: - dependency-name: github.com/nwaples/rardecode/v2 dependency-version: 2.2.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e09106058..5ce199f6a 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect - github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect + github.com/nwaples/rardecode/v2 v2.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect diff --git a/go.sum b/go.sum index e52607b63..c415b8251 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= -github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= -github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= +github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From 8117b1acf3f46f087c55f0890eca0c3a2c17051f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 16 Oct 2025 14:56:57 +0300 Subject: [PATCH 080/132] add windvane --- pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 2 + pkg/subscraping/sources/windvane/windvane.go | 146 +++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 pkg/subscraping/sources/windvane/windvane.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index e74d874be..44be7b8d6 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -51,6 +51,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -95,6 +96,7 @@ var AllSources = [...]subscraping.Source{ &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, + &windvane.Source{}, &zoomeyeapi.Source{}, &facebook.Source{}, // &threatminer.Source{}, // failing api diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 4febbf9fa..a90ff8111 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -48,6 +48,7 @@ var ( "virustotal", "waybackarchive", "whoisxmlapi", + "windvane", "zoomeyeapi", "hunter", "leakix", @@ -85,6 +86,7 @@ var ( "rsecloud", "securitytrails", "shodan", + "windvane", "virustotal", "whoisxmlapi", "hunter", diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go new file mode 100644 index 000000000..194804d9b --- /dev/null +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -0,0 +1,146 @@ +// Package windvane logic +package windvane + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data responseData `json:"data"` +} + +type responseData struct { + List []domainEntry `json:"list"` + PageResponse pageInfo `json:"page_response"` +} + +type domainEntry struct { + Domain string `json:"domain"` +} + +type pageInfo struct { + Total string `json:"total"` + Count string `json:"count"` + TotalPage string `json:"total_page"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "X-Api-Key": randomApiKey} + + page := 1 + count := 1000 + for { + var resp *http.Response + var err error + + requestBody := fmt.Appendf(nil, `{"domain":"%s","page_request":{"page":%d,"count":%d}}`, domain, page, count) + resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", + "", headers, bytes.NewReader(requestBody)) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer session.DiscardHTTPResponse(resp) + + var windvaneResponse response + err = json.NewDecoder(resp.Body).Decode(&windvaneResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + for _, record := range windvaneResponse.Data.List { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} + s.results++ + } + + pageInfo := windvaneResponse.Data.PageResponse + var totalRecords, recordsPerPage int + + if totalRecords, err = strconv.Atoi(pageInfo.Total); err != nil { + break + } + if recordsPerPage, err = strconv.Atoi(pageInfo.Count); err != nil { + break + } + + if (page-1)*recordsPerPage >= totalRecords { + break + } + + page++ + } + + }() + + return results +} + +func (s *Source) Name() string { + return "windvane" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 26a221fffeebcf57a6d24984fd9264818b62d64b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 19 Oct 2025 15:03:08 +0400 Subject: [PATCH 081/132] using json marshal --- pkg/subscraping/sources/windvane/windvane.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go index 194804d9b..768cc3fd0 100644 --- a/pkg/subscraping/sources/windvane/windvane.go +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/json" - "fmt" "net/http" "strconv" "time" @@ -67,7 +66,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var resp *http.Response var err error - requestBody := fmt.Appendf(nil, `{"domain":"%s","page_request":{"page":%d,"count":%d}}`, domain, page, count) + requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}}) resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", "", headers, bytes.NewReader(requestBody)) From 61bc082d5d72f1c6d829c6f21df43d61aa764ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 20 Oct 2025 11:27:24 +0300 Subject: [PATCH 082/132] add api key support --- .../sources/alienvault/alienvault.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go index 27e1f57b4..c4c983669 100644 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -23,6 +23,8 @@ type Source struct { timeTaken time.Duration results int errors int + apiKeys []string + skipped bool } // Run function returns all subdomains found with the service @@ -37,7 +39,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "", + map[string]string{"Authorization": "Bearer " + randomApiKey}) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -86,11 +95,11 @@ func (s *Source) HasRecursiveSupport() bool { } func (s *Source) NeedsKey() bool { - return false + return true } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { @@ -98,5 +107,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Skipped: s.skipped, } } From bc7edda8b6fa738de646f1c6fa8c872e07e2bc98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 20 Oct 2025 11:31:10 +0300 Subject: [PATCH 083/132] remove hunter --- .github/workflows/build-test.yml | 1 - pkg/passive/sources.go | 2 - pkg/passive/sources_test.go | 2 - pkg/subscraping/sources/hunter/hunter.go | 131 ----------------------- 4 files changed, 136 deletions(-) delete mode 100644 pkg/subscraping/sources/hunter/hunter.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e6fabe953..d567636b7 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -58,7 +58,6 @@ jobs: FOFA_API_KEY: ${{secrets.FOFA_API_KEY}} FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}} GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}} - HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}} INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 2f1a52217..af7d5e6e8 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -33,7 +33,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" @@ -78,7 +77,6 @@ var AllSources = [...]subscraping.Source{ &fullhunt.Source{}, &github.Source{}, &hackertarget.Source{}, - &hunter.Source{}, &intelx.Source{}, &netlas.Source{}, &onyphe.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index e0a2e9ac2..abcad1471 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -51,7 +51,6 @@ var ( "whoisxmlapi", "windvane", "zoomeyeapi", - "hunter", "leakix", "facebook", // "threatminer", @@ -91,7 +90,6 @@ var ( "windvane", "virustotal", "whoisxmlapi", - "hunter", "leakix", "facebook", // "threatminer", diff --git a/pkg/subscraping/sources/hunter/hunter.go b/pkg/subscraping/sources/hunter/hunter.go deleted file mode 100644 index 960563279..000000000 --- a/pkg/subscraping/sources/hunter/hunter.go +++ /dev/null @@ -1,131 +0,0 @@ -package hunter - -import ( - "context" - "encoding/base64" - "fmt" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -type hunterResp struct { - Code int `json:"code"` - Data hunterData `json:"data"` - Message string `json:"message"` -} - -type infoArr struct { - URL string `json:"url"` - IP string `json:"ip"` - Port int `json:"port"` - Domain string `json:"domain"` - Protocol string `json:"protocol"` -} - -type hunterData struct { - InfoArr []infoArr `json:"arr"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []string - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } - - var pages = 1 - for currentPage := 1; currentPage <= pages; currentPage++ { - // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 - qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage)) - if err != nil && resp == nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var response hunterResp - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - session.DiscardHTTPResponse(resp) - - if response.Code == 401 || response.Code == 400 { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), - } - s.errors++ - return - } - - if response.Data.Total > 0 { - for _, hunterInfo := range response.Data.InfoArr { - subdomain := hunterInfo.Domain - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - } - pages = int(response.Data.Total/1000) + 1 - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "hunter" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} From 02ea9f8e45555f2df7956acdc9fdb4d1357cb15e Mon Sep 17 00:00:00 2001 From: Bohdan Turkynewych Date: Mon, 20 Oct 2025 16:29:34 -0400 Subject: [PATCH 084/132] New source: DomainsProject --- pkg/passive/sources.go | 2 + .../sources/domainsproject/domainsproject.go | 131 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 pkg/subscraping/sources/domainsproject/domainsproject.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index af7d5e6e8..e6f5538a6 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -25,6 +25,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" @@ -71,6 +72,7 @@ var AllSources = [...]subscraping.Source{ &digitorus.Source{}, &dnsdb.Source{}, &dnsdumpster.Source{}, + &domainsproject.Source{}, &dnsrepo.Source{}, &driftnet.Source{}, &fofa.Source{}, diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go new file mode 100644 index 000000000..ac4d8a0db --- /dev/null +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -0,0 +1,131 @@ +// Package domainsproject logic +package domainsproject + +import ( + "context" + "fmt" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey + timeTaken time.Duration + errors int + results int + skipped bool +} + +type apiKey struct { + username string + password string +} + +type domainsProjectResponse struct { + Domains []string `json:"domains"` + Error string `json:"error"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.username == "" || randomApiKey.password == "" { + s.skipped = true + return + } + + searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain) + resp, err := session.HTTPRequest( + ctx, + "GET", + searchURL, + "", + nil, + nil, + subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, + ) + if err != nil { + session.DiscardHTTPResponse(resp) + return + } + + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + var response domainsProjectResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if response.Error != "" { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), + } + s.errors++ + return + } + + for _, subdomain := range response.Domains { + if !strings.HasPrefix(subdomain, ".") { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "domainsproject" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From c1b64d87353e7dee74dbace22ca2fc863170a643 Mon Sep 17 00:00:00 2001 From: Bohdan Turkynewych Date: Mon, 20 Oct 2025 16:36:48 -0400 Subject: [PATCH 085/132] CR fixes --- pkg/subscraping/sources/domainsproject/domainsproject.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go index ac4d8a0db..4ae8d00dc 100644 --- a/pkg/subscraping/sources/domainsproject/domainsproject.go +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -60,6 +60,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, ) if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ session.DiscardHTTPResponse(resp) return } From 3ba0a308fa034121edc8f1395b9aa9757cef9146 Mon Sep 17 00:00:00 2001 From: Bohdan Turkynewych Date: Mon, 20 Oct 2025 16:47:21 -0400 Subject: [PATCH 086/132] Add domainsproject to sources tests --- pkg/passive/sources_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index abcad1471..94c588160 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -26,6 +26,7 @@ var ( "dnsdumpster", "dnsdb", "dnsrepo", + "domainsproject", "driftnet", "fofa", "fullhunt", @@ -73,6 +74,7 @@ var ( "crtsh", "digitorus", "dnsdumpster", + "domainsproject", "dnsrepo", "driftnet", "fofa", From 3cb054d97694e1a440725b27bfabee73563198a0 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Wed, 5 Nov 2025 21:49:07 +0530 Subject: [PATCH 087/132] Add wildcard certificate detection in JSON output (#1665) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add wildcard certificate detection in JSON output This commit implements wildcard certificate detection for subdomains that are covered by wildcard certificates (e.g., *.example.com). When sources return results containing wildcard patterns, the subdomain is now marked with a wildcard_certificate field in JSON output mode. Key changes: - Added WildcardCertificate field to HostEntry and Result structs - Detection logic: checks if result.Value contains "*.subdomain" pattern - Propagates wildcard flag through resolution pipeline - JSON output includes wildcard_certificate field (omitted if false) - Updates version to v2.9.1-dev - Fix .goreleaser.yml 386 architecture quoting - Add /subfinder to .gitignore Note: wildcard_certificate field is not included when using -cs flag to avoid breaking changes to the library API. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * Fix wildcard certificate flag propagation with -nW flag Fixed a bug where the wildcard_certificate field was being lost when using the -nW flag (DNS resolution with wildcard filtering). The issue occurred because when multiple sources find the same subdomain, only the first occurrence is sent to the resolution pool. If a later source marks the subdomain as having a wildcard certificate, that information was stored in uniqueMap but never propagated to foundResults. Solution: After resolution completes, merge wildcard certificate information from uniqueMap into foundResults. This ensures that if any source marked a subdomain as having a wildcard certificate, that flag is preserved in the final output. This ensures consistent behavior - wildcard_certificate field appears in JSON output regardless of whether -nW flag is used. Validation: - Without -nW: api.nuclei.sh shows wildcard_certificate:true βœ“ - With -nW: api.nuclei.sh shows wildcard_certificate:true βœ“ πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --------- Co-authored-by: Claude --- .gitignore | 3 ++- .goreleaser.yml | 2 +- pkg/resolve/resolve.go | 24 +++++++++++++----------- pkg/runner/banners.go | 2 +- pkg/runner/enumerate.go | 25 +++++++++++++++++++++++-- pkg/runner/outputter.go | 28 ++++++++++++++++------------ 6 files changed, 56 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index b0dd47ba3..155676bfe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ vendor/ .idea .devcontainer .vscode -dist \ No newline at end of file +dist +/subfinder \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 867267d2e..e34562f04 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -13,7 +13,7 @@ builds: - darwin goarch: - amd64 - - 386 + - '386' - arm - arm64 diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go index d7e4d887c..38055b89b 100644 --- a/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -25,18 +25,20 @@ type ResolutionPool struct { // HostEntry defines a host with the source type HostEntry struct { - Domain string - Host string - Source string + Domain string + Host string + Source string + WildcardCertificate bool } // Result contains the result for a host resolution type Result struct { - Type ResultType - Host string - IP string - Error error - Source string + Type ResultType + Host string + IP string + Error error + Source string + WildcardCertificate bool } // ResultType is the type of result found @@ -92,13 +94,13 @@ func (r *ResolutionPool) InitWildcards(domain string) error { func (r *ResolutionPool) resolveWorker() { for task := range r.Tasks { if !r.removeWildcard { - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source, WildcardCertificate: task.WildcardCertificate} continue } hosts, err := r.DNSClient.Lookup(task.Host) if err != nil { - r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} + r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err, WildcardCertificate: task.WildcardCertificate} continue } @@ -116,7 +118,7 @@ func (r *ResolutionPool) resolveWorker() { } if !skip { - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source, WildcardCertificate: task.WildcardCertificate} } } r.wg.Done() diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index d02334782..7e52f2b66 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.9.0` +const version = `v2.9.1-dev` // showBanner is used to show the banner to the user func showBanner() { diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index f39eab068..bd8953e56 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -67,6 +67,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error) case subscraping.Subdomain: subdomain := replacer.Replace(result.Value) + // check if this subdomain is actually a wildcard subdomain + // that may have furthur subdomains associated with it + isWildcard := strings.Contains(result.Value, "*."+subdomain) // Validate the subdomain found and remove wildcards from if !strings.HasSuffix(subdomain, "."+domain) { @@ -90,10 +93,17 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // send the subdomain for resolution. if _, ok := uniqueMap[subdomain]; ok { skippedCounts[result.Source]++ + // even if it is duplicate if it was not marked as wildcard before but this source says it is wildcard + // then we should mark it as wildcard + if !uniqueMap[subdomain].WildcardCertificate && isWildcard { + val := uniqueMap[subdomain] + val.WildcardCertificate = true + uniqueMap[subdomain] = val + } continue } - hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source} + hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source, WildcardCertificate: isWildcard} if r.options.ResultCallback != nil && !r.options.RemoveWildcard { r.options.ResultCallback(&hostEntry) } @@ -112,6 +122,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string if r.options.RemoveWildcard { close(resolutionPool.Tasks) } + wg.Done() }() @@ -129,11 +140,21 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string if _, ok := foundResults[result.Host]; !ok { foundResults[result.Host] = result if r.options.ResultCallback != nil { - r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source}) + r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate}) } } } } + + // Merge wildcard certificate information from uniqueMap into foundResults + // This handles cases where a later source marked a subdomain as wildcard + // after it was already sent to the resolution pool + for host, result := range foundResults { + if entry, ok := uniqueMap[host]; ok && entry.WildcardCertificate && !result.WildcardCertificate { + result.WildcardCertificate = true + foundResults[host] = result + } + } } wg.Wait() outputWriter := NewOutputWriter(r.options.JSON) diff --git a/pkg/runner/outputter.go b/pkg/runner/outputter.go index cfa4fc1a5..e3f1839c2 100644 --- a/pkg/runner/outputter.go +++ b/pkg/runner/outputter.go @@ -19,22 +19,25 @@ type OutputWriter struct { } type jsonSourceResult struct { - Host string `json:"host"` - Input string `json:"input"` - Source string `json:"source"` + Host string `json:"host"` + Input string `json:"input"` + Source string `json:"source"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourceIPResult struct { - Host string `json:"host"` - IP string `json:"ip"` - Input string `json:"input"` - Source string `json:"source"` + Host string `json:"host"` + IP string `json:"ip"` + Input string `json:"input"` + Source string `json:"source"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourcesResult struct { - Host string `json:"host"` - Input string `json:"input"` - Sources []string `json:"sources"` + Host string `json:"host"` + Input string `json:"input"` + Sources []string `json:"sources"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } // NewOutputWriter creates a new OutputWriter @@ -117,7 +120,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io. data.IP = result.IP data.Input = input data.Source = result.Source - + data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(&data) if err != nil { return err @@ -130,7 +133,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io. func (o *OutputWriter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error { hosts := make(map[string]resolve.HostEntry) for host, result := range results { - hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source} + hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate} } return o.WriteHost(input, hosts, writer) @@ -175,6 +178,7 @@ func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io data.Host = result.Host data.Input = input data.Source = result.Source + data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(data) if err != nil { return err From d5255517a32122966f72bc92796fac589564b770 Mon Sep 17 00:00:00 2001 From: sunnyraindy Date: Mon, 10 Nov 2025 17:49:28 +0800 Subject: [PATCH 088/132] refactor: use strings.Builder to improve performance Signed-off-by: sunnyraindy --- pkg/runner/outputter.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/runner/outputter.go b/pkg/runner/outputter.go index e3f1839c2..2a41f5127 100644 --- a/pkg/runner/outputter.go +++ b/pkg/runner/outputter.go @@ -227,11 +227,12 @@ func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, wr for host, sources := range sourceMap { sb.WriteString(host) sb.WriteString(",[") - sourcesString := "" + var sourcesString strings.Builder for source := range sources { - sourcesString += source + "," + sourcesString.WriteString(source) + sourcesString.WriteRune(',') } - sb.WriteString(strings.Trim(sourcesString, ", ")) + sb.WriteString(strings.TrimSuffix(sourcesString.String(), ",")) sb.WriteString("]\n") _, err := bufwriter.WriteString(sb.String()) From deeaf0cf1fcd2b70717d65b588bdcc51a7d2f4d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 02:01:05 +0000 Subject: [PATCH 089/132] chore(deps): bump golang.org/x/crypto from 0.36.0 to 0.45.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.36.0 to 0.45.0. - [Commits](https://github.com/golang/crypto/compare/v0.36.0...v0.45.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 5ce199f6a..8ef3f30a1 100644 --- a/go.mod +++ b/go.mod @@ -111,14 +111,14 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/mod v0.22.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/mod v0.29.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.29.0 // indirect + golang.org/x/tools v0.38.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) @@ -133,6 +133,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 github.com/projectdiscovery/retryabledns v1.0.102 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.31.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index c415b8251..e56e41e07 100644 --- a/go.sum +++ b/go.sum @@ -415,8 +415,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -446,8 +446,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -472,8 +472,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -491,8 +491,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -527,8 +527,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -536,8 +536,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -550,8 +550,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -582,8 +582,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From c83e0b62b4af1eddc052acafc730c806c23defb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 21 Nov 2025 13:09:54 +0900 Subject: [PATCH 090/132] bump version --- pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index 7e52f2b66..c7768520f 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.9.1-dev` +const version = `v2.10.1` // showBanner is used to show the banner to the user func showBanner() { From 786a6464b5339b7de56c2b572cfbbe83f6e7bbeb Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Sat, 22 Nov 2025 01:42:56 +0900 Subject: [PATCH 091/132] v2.10.1 (#1674) * chore(deps): bump golang.org/x/crypto from 0.36.0 to 0.45.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.36.0 to 0.45.0. - [Commits](https://github.com/golang/crypto/compare/v0.36.0...v0.45.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] * bump version --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- pkg/runner/banners.go | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 5ce199f6a..8ef3f30a1 100644 --- a/go.mod +++ b/go.mod @@ -111,14 +111,14 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/mod v0.22.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/mod v0.29.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.29.0 // indirect + golang.org/x/tools v0.38.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) @@ -133,6 +133,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 github.com/projectdiscovery/retryabledns v1.0.102 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.31.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index c415b8251..e56e41e07 100644 --- a/go.sum +++ b/go.sum @@ -415,8 +415,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -446,8 +446,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -472,8 +472,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -491,8 +491,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -527,8 +527,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -536,8 +536,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -550,8 +550,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -582,8 +582,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index 7e52f2b66..c7768520f 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.9.1-dev` +const version = `v2.10.1` // showBanner is used to show the banner to the user func showBanner() { From deccd62f598f5bee7358174c849502cdeae76e32 Mon Sep 17 00:00:00 2001 From: Nakul Bharti Date: Thu, 4 Dec 2025 23:04:40 +0530 Subject: [PATCH 092/132] fix:context cancellation sources (#1680) * fix: context cancellation for the major sources * fix: handle context cancellation for the minor sources * fix: handle context cancellation for the minor sources --- pkg/passive/passive.go | 8 +++- .../sources/alienvault/alienvault.go | 8 +++- pkg/subscraping/sources/anubis/anubis.go | 8 +++- pkg/subscraping/sources/bevigil/bevigil.go | 8 +++- .../sources/bufferover/bufferover.go | 8 +++- .../sources/builtwith/builtwith.go | 8 +++- pkg/subscraping/sources/c99/c99.go | 8 +++- pkg/subscraping/sources/censys/censys.go | 14 ++++-- .../sources/certspotter/certspotter.go | 22 +++++++-- pkg/subscraping/sources/chaos/chaos.go | 7 ++- pkg/subscraping/sources/chinaz/chinaz.go | 13 +++++- .../sources/commoncrawl/commoncrawl.go | 16 +++++-- pkg/subscraping/sources/crtsh/crtsh.go | 27 +++++++++-- .../sources/digitalyama/digitalyama.go | 8 +++- .../sources/digitorus/digitorus.go | 13 ++++-- pkg/subscraping/sources/dnsdb/dnsdb.go | 26 +++++++---- .../sources/dnsdumpster/dnsdumpster.go | 8 +++- pkg/subscraping/sources/dnsrepo/dnsrepo.go | 8 ++-- .../sources/domainsproject/domainsproject.go | 8 +++- pkg/subscraping/sources/driftnet/driftnet.go | 17 +++++-- pkg/subscraping/sources/facebook/ctlogs.go | 15 ++++-- pkg/subscraping/sources/fofa/fofa.go | 5 ++ pkg/subscraping/sources/fullhunt/fullhunt.go | 8 +++- pkg/subscraping/sources/github/github.go | 46 ++++++++++++++++--- pkg/subscraping/sources/gitlab/gitlab.go | 7 ++- .../sources/hackertarget/hackertarget.go | 13 +++++- .../sources/hudsonrock/hudsonrock.go | 8 +++- pkg/subscraping/sources/intelx/intelx.go | 13 ++++-- pkg/subscraping/sources/leakix/leakix.go | 8 ++-- pkg/subscraping/sources/netlas/netlas.go | 8 ++-- pkg/subscraping/sources/onyphe/onyphe.go | 10 ++++ pkg/subscraping/sources/pugrecon/pugrecon.go | 8 ++-- pkg/subscraping/sources/quake/quake.go | 10 ++++ pkg/subscraping/sources/rapiddns/rapiddns.go | 13 +++++- .../sources/reconcloud/reconcloud.go | 8 +++- .../sources/redhuntlabs/redhuntlabs.go | 21 +++++++-- pkg/subscraping/sources/riddler/riddler.go | 15 +++++- pkg/subscraping/sources/robtex/robtext.go | 13 +++++- pkg/subscraping/sources/rsecloud/rsecloud.go | 13 +++++- .../sources/securitytrails/securitytrails.go | 18 +++++++- pkg/subscraping/sources/shodan/shodan.go | 10 ++++ .../sources/sitedossier/sitedossier.go | 8 +++- .../sources/threatbook/threatbook.go | 8 +++- .../sources/threatcrowd/threatcrowd.go | 8 +++- .../sources/threatminer/threatminer.go | 8 +++- .../sources/virustotal/virustotal.go | 13 +++++- .../sources/waybackarchive/waybackarchive.go | 14 ++++-- .../sources/whoisxmlapi/whoisxmlapi.go | 8 +++- pkg/subscraping/sources/windvane/windvane.go | 13 +++++- .../sources/zoomeyeapi/zoomeyeapi.go | 13 +++++- 50 files changed, 482 insertions(+), 124 deletions(-) diff --git a/pkg/passive/passive.go b/pkg/passive/passive.go index 199686bd4..81a7cf041 100644 --- a/pkg/passive/passive.go +++ b/pkg/passive/passive.go @@ -65,11 +65,15 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p for _, runner := range a.sources { wg.Add(1) go func(source subscraping.Source) { + defer wg.Done() ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { - results <- resp + select { + case <-ctx.Done(): + return + case results <- resp: + } } - wg.Done() }(runner) } wg.Wait() diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go index c4c983669..6af4268f2 100644 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -73,8 +73,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, record := range response.PassiveDNS { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go index 6efbd5343..96d21e784 100644 --- a/pkg/subscraping/sources/anubis/anubis.go +++ b/pkg/subscraping/sources/anubis/anubis.go @@ -56,8 +56,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go index 2021e4c64..cbdcf0f43 100644 --- a/pkg/subscraping/sources/bevigil/bevigil.go +++ b/pkg/subscraping/sources/bevigil/bevigil.go @@ -70,8 +70,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 207cb8190..198d31e0b 100644 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -96,8 +96,12 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s for _, subdomain := range subdomains { for _, value := range session.Extractor.Extract(subdomain) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } diff --git a/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go index 4c6b516f6..a22043375 100644 --- a/pkg/subscraping/sources/builtwith/builtwith.go +++ b/pkg/subscraping/sources/builtwith/builtwith.go @@ -74,8 +74,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, result := range data.Results { for _, path := range result.Result.Paths { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go index c2e45e818..124686d27 100644 --- a/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -81,8 +81,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, data := range response.Subdomains { if !strings.HasPrefix(data.Subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 4427821ae..b5c17a8b4 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -89,6 +89,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se cursor := "" currentPage := 1 for { + select { + case <-ctx.Done(): + return + default: + } certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -132,12 +137,15 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, hit := range censysResponse.Result.Hits { for _, name := range hit.Names { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}: + s.results++ + } } } - // Exit the censys enumeration if last page is reached cursor = censysResponse.Result.Links.Next if cursor == "" || currentPage >= maxCensysPages { break diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index e501e1730..35118ff65 100644 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -66,18 +66,26 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, cert := range response { for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } - // if the number of responses is zero, close the channel and return. if len(response) == 0 { return } id := response[len(response)-1].ID for { + select { + case <-ctx.Done(): + return + default: + } reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) resp, err := session.Get(ctx, reqURL, cookies, headers) @@ -103,8 +111,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, cert := range response { for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } diff --git a/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go index 2cc873679..489e3173a 100644 --- a/pkg/subscraping/sources/chaos/chaos.go +++ b/pkg/subscraping/sources/chaos/chaos.go @@ -20,7 +20,7 @@ type Source struct { } // Run function returns all subdomains found with the service -func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { +func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 @@ -41,6 +41,11 @@ func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) < for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ Domain: domain, }) { + select { + case <-ctx.Done(): + return + default: + } if result.Error != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} s.errors++ diff --git a/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go index c6e43c05d..2fd6c9546 100644 --- a/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -55,9 +55,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if SubdomainList.ToBool() { _data := []byte(SubdomainList.ToString()) for i := 0; i < SubdomainList.Size(); i++ { + select { + case <-ctx.Done(): + return + default: + } subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } else { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index 09ad4967d..b1dda4910 100644 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -138,6 +138,12 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return false + default: + } line := scanner.Text() if line == "" { continue @@ -145,13 +151,17 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { if subdomain != "" { - // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return false + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } } diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index 76ac02b45..caea79c42 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -114,8 +114,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio var count int var data string - // Parse all the rows getting subdomains for rows.Next() { + select { + case <-ctx.Done(): + return count + default: + } err := rows.Scan(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -127,8 +131,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio for subdomain := range strings.SplitSeq(data, "\n") { for _, value := range session.Extractor.Extract(subdomain) { if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return count + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -157,11 +165,20 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi session.DiscardHTTPResponse(resp) for _, subdomain := range subdomains { + select { + case <-ctx.Done(): + return true + default: + } for sub := range strings.SplitSeq(subdomain.NameValue, "\n") { for _, value := range session.Extractor.Extract(sub) { if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return true + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } diff --git a/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go index 7ab719576..8a89e686e 100644 --- a/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -94,8 +94,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go index 8c680a2b6..91b3ca5b3 100644 --- a/pkg/subscraping/sources/digitorus/digitorus.go +++ b/pkg/subscraping/sources/digitorus/digitorus.go @@ -50,16 +50,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } subdomains := session.Extractor.Extract(line) for _, subdomain := range subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."), + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, ".")}: + s.results++ } - s.results++ } } }() diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go index e5dc0e246..810d8a05b 100644 --- a/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -85,6 +85,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se queryParams.Add("swclient", "subfinder") for { + select { + case <-ctx.Done(): + return + default: + } url := urlTemplate + queryParams.Encode() resp, err := session.Get(ctx, url, "", headers) @@ -98,6 +103,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var respCond string reader := bufio.NewReader(resp.Body) for { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } n, err := reader.ReadBytes('\n') if err == io.EOF { break @@ -117,21 +128,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // Condition is a scalar enum of string values: {β€œbegin”, β€œongoing”, β€œsucceeded”, β€œlimited”, β€œfailed”}. - // "begin" will be the initiating Condition, this can be safely ignored. The data of interest will be in - // objects with Condition "" or "ongoing". Conditions "succeeded", "limited", and "failed" are terminating conditions. - // See https://www.domaintools.com/resources/user-guides/farsight-streaming-api-framing-protocol-documentation/ - // for more details respCond = response.Condition if respCond == "" || respCond == "ongoing" { if response.Obj.Name != "" { - results <- subscraping.Result{ - Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, "."), + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, ".")}: + s.results++ } - s.results++ } } else if respCond != "begin" { - // if the respCond is not "", "ongoing", or "begin", then it is a terminating condition, so break out of the loop break } } diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 72be41c3b..8062a4954 100644 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -65,8 +65,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, record := range append(response.A, response.Ns...) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go index e98c82399..f7faaaac0 100644 --- a/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -74,10 +74,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, sub := range result { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, "."), + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")}: + s.results++ } - s.results++ } }() diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go index 4ae8d00dc..800727d78 100644 --- a/pkg/subscraping/sources/domainsproject/domainsproject.go +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -91,8 +91,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, subdomain := range response.Domains { if !strings.HasPrefix(subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index 758bbfa44..fa0d0bcdd 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -176,17 +176,24 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc } for subdomain := range summary.Summary.Values { - // We can get certificate results which aren't actually subdomains of the target domain. Skip them. + select { + case <-ctx.Done(): + wg.Done() + return + default: + } if !strings.HasSuffix(subdomain, "."+domain) { continue } - // Avoid returning the same result more than once from the same source (can happen as we are using multiple endpoints) if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain, + select { + case <-ctx.Done(): + wg.Done() + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) } - s.results.Add(1) } } diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index f96436008..f0f37a4e0 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -103,7 +103,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se domainsURL := fmt.Sprintf(domainsUrl, key.AccessToken, domain) for { - // unfortunately, this cannot be parllelized since pagination is cursor based + select { + case <-ctx.Done(): + return + default: + } resp, err := session.Get(ctx, domainsURL, "", nil) if err != nil { s.errors++ @@ -126,14 +130,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, v := range response.Data { for _, domain := range v.Domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}: + s.results++ + } } } if response.Paging.Next == "" { break } - // cursor includes api key so no need to update it domainsURL = updateParamInURL(response.Paging.Next, "limit", domainsPerPage) } }() diff --git a/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go index 78be69d1f..b885476aa 100644 --- a/pkg/subscraping/sources/fofa/fofa.go +++ b/pkg/subscraping/sources/fofa/fofa.go @@ -83,6 +83,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if response.Size > 0 { for _, subdomain := range response.Results { + select { + case <-ctx.Done(): + return + default: + } if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { subdomain = subdomain[strings.Index(subdomain, "//")+2:] } diff --git a/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go index 60c64e52d..c53206e11 100644 --- a/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -61,8 +61,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } session.DiscardHTTPResponse(resp) for _, record := range response.Hosts { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go index 2e8b3224b..69f5e151c 100644 --- a/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -129,6 +129,11 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * linksHeader := linkheader.Parse(resp.Header.Get("Link")) // Process the next link recursively for _, link := range linksHeader { + select { + case <-ctx.Done(): + return + default: + } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { @@ -147,11 +152,21 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r errChan := make(chan error, len(items)) for _, responseItem := range items { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } wg.Add(1) go func(responseItem item) { defer wg.Done() - // find subdomains in code + select { + case <-ctx.Done(): + return + default: + } + resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { @@ -164,23 +179,42 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r if resp.StatusCode == http.StatusOK { scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } line := scanner.Text() if line == "" { continue } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } session.DiscardHTTPResponse(resp) } - // find subdomains in text matches for _, textMatch := range responseItem.TextMatches { + select { + case <-ctx.Done(): + return + default: + } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }(responseItem) diff --git a/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go index 902ef4691..e2a13a111 100644 --- a/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -120,10 +120,13 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * }(it) } - // Links header, first, next, last... linksHeader := linkheader.Parse(resp.Header.Get("Link")) - // Process the next link recursively for _, link := range linksHeader { + select { + case <-ctx.Done(): + return + default: + } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index c84957722..92d898c9a 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -46,14 +46,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } match := session.Extractor.Extract(line) for _, subdomain := range match { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go index 51d6bc2fa..b3e2e0ec8 100644 --- a/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -60,8 +60,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) { for _, subdomain := range session.Extractor.Extract(record.URL) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go index b3a3a45a1..82ecaeb16 100644 --- a/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -107,6 +107,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) status := 0 for status == 0 || status == 3 { + select { + case <-ctx.Done(): + return + default: + } resp, err = session.Get(ctx, resultsURL, "", nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -134,10 +139,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se status = response.Status for _, hostname := range response.Selectors { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}: + s.results++ } - s.results++ } } }() diff --git a/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go index 4159e6aba..0d3526ab7 100644 --- a/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -64,10 +64,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, result := range subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: + s.results++ } - s.results++ } }() return results diff --git a/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go index cd9f612e6..a0c8b86e7 100644 --- a/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -158,10 +158,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, item := range data { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain}: + s.results++ } - s.results++ } }() diff --git a/pkg/subscraping/sources/onyphe/onyphe.go b/pkg/subscraping/sources/onyphe/onyphe.go index 6ea1dcbb1..a304ebdb3 100644 --- a/pkg/subscraping/sources/onyphe/onyphe.go +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -63,6 +63,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se pageSize := 1000 for { + select { + case <-ctx.Done(): + return + default: + } var resp *http.Response var err error @@ -89,6 +94,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range respOnyphe.Results { + select { + case <-ctx.Done(): + return + default: + } for _, subdomain := range record.Subdomains { if subdomain != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} diff --git a/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go index e7f2255f8..855154153 100644 --- a/pkg/subscraping/sources/pugrecon/pugrecon.go +++ b/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -106,10 +106,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range response.Results { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name}: + s.results++ } - s.results++ } }() diff --git a/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go index 5cac4ed2e..2e99fb87f 100644 --- a/pkg/subscraping/sources/quake/quake.go +++ b/pkg/subscraping/sources/quake/quake.go @@ -63,6 +63,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var totalResults = -1 for { + select { + case <-ctx.Done(): + return + default: + } var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, @@ -97,6 +102,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, quakeDomain := range response.Data { + select { + case <-ctx.Done(): + return + default: + } subdomain := quakeDomain.Service.HTTP.Host if strings.ContainsAny(subdomain, "ζš‚ζ— ζƒι™") { continue diff --git a/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go index 89ec70632..eb7969795 100644 --- a/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -36,6 +36,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 maxPages := 1 for { + select { + case <-ctx.Done(): + return + default: + } resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -56,8 +61,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se src := string(body) for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } if maxPages == 1 { diff --git a/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go index 8aacc7d4e..61bb593da 100644 --- a/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -63,8 +63,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if len(response.CloudAssetsList) > 0 { for _, cloudAsset := range response.CloudAssetsList { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index a07e9a5b4..009d5d000 100644 --- a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -76,6 +76,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if response.Metadata.ResultCount > pageSize { totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize for page := 1; page <= totalPages; page++ { + select { + case <-ctx.Done(): + return + default: + } getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { @@ -96,14 +101,22 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } } else { for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } diff --git a/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go index 9a3401c27..7855a344b 100644 --- a/pkg/subscraping/sources/riddler/riddler.go +++ b/pkg/subscraping/sources/riddler/riddler.go @@ -39,13 +39,24 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } line := scanner.Text() if line == "" { continue } for _, subdomain := range session.Extractor.Extract(line) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } session.DiscardHTTPResponse(resp) diff --git a/pkg/subscraping/sources/robtex/robtext.go b/pkg/subscraping/sources/robtex/robtext.go index 9214363ed..f049d97ed 100644 --- a/pkg/subscraping/sources/robtex/robtext.go +++ b/pkg/subscraping/sources/robtex/robtext.go @@ -62,6 +62,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, result := range ips { + select { + case <-ctx.Done(): + return + default: + } if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers) if err != nil { @@ -70,8 +75,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, result := range domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}: + s.results++ + } } } } diff --git a/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go index 8601b7732..43d270b05 100644 --- a/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -50,6 +50,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se fetchSubdomains := func(endpoint string) { page := 1 for { + select { + case <-ctx.Done(): + return + default: + } resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -68,8 +73,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range rseCloudResponse.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } if page >= rseCloudResponse.TotalPages { diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index e6d38fe44..0e6846e55 100644 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -56,6 +56,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} for { + select { + case <-ctx.Done(): + return + default: + } var resp *http.Response var err error @@ -90,11 +95,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range securityTrailsResponse.Records { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: + s.results++ + } } for _, subdomain := range securityTrailsResponse.Subdomains { + select { + case <-ctx.Done(): + return + default: + } if strings.HasSuffix(subdomain, ".") { subdomain += domain } else { diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index d2224a810..b2195e393 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -48,6 +48,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 for { + select { + case <-ctx.Done(): + return + default: + } searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) resp, err := session.SimpleGet(ctx, searchURL) @@ -75,6 +80,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, data := range response.Subdomains { + select { + case <-ctx.Done(): + return + default: + } value := fmt.Sprintf("%s.%s", data, response.Domain) results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: value, diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index 38d31dd5c..dde639c40 100644 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -70,8 +70,12 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba src := string(body) for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } match := reNext.FindStringSubmatch(src) diff --git a/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go index 0033a4d32..6a6645d3b 100644 --- a/pkg/subscraping/sources/threatbook/threatbook.go +++ b/pkg/subscraping/sources/threatbook/threatbook.go @@ -87,8 +87,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if total > 0 { for _, subdomain := range response.Data.SubDomains.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index e09ab91ef..831659f61 100644 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -80,8 +80,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, subdomain := range tcResponse.Subdomains { if subdomain != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }(time.Now()) diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 6776f9ef9..77f5a586e 100644 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -55,8 +55,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range data.Results { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 7ac4c6c50..446c7c309 100644 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -51,6 +51,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var cursor = "" for { + select { + case <-ctx.Done(): + return + default: + } var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) @@ -78,8 +83,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range data.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}: + s.results++ + } } cursor = data.Meta.Cursor if cursor == "" { diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index f3cba9809..ecbe4d943 100644 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -43,19 +43,27 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { - // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() diff --git a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index 18375e3ff..c4efbdd2e 100644 --- a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -74,8 +74,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range data.Result.Records { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: + s.results++ + } } }() diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go index 768cc3fd0..92f972c40 100644 --- a/pkg/subscraping/sources/windvane/windvane.go +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -63,6 +63,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 count := 1000 for { + select { + case <-ctx.Done(): + return + default: + } var resp *http.Response var err error @@ -89,8 +94,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, record := range windvaneResponse.Data.List { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: + s.results++ + } } pageInfo := windvaneResponse.Data.PageResponse diff --git a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 62c2d1f91..70d491c60 100644 --- a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -63,6 +63,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var pages = 1 for currentPage := 1; currentPage <= pages; currentPage++ { + select { + case <-ctx.Done(): + return + default: + } api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden @@ -87,8 +92,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se _ = resp.Body.Close() pages = int(res.Total/1000) + 1 for _, r := range res.List { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}: + s.results++ + } } } }() From 074497b5366c5589802cce1561325077fde94429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 8 Dec 2025 10:31:36 +0300 Subject: [PATCH 093/132] add profundis --- pkg/passive/sources.go | 4 +- pkg/passive/sources_test.go | 2 + .../sources/profundis/profundis.go | 127 ++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 pkg/subscraping/sources/profundis/profundis.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index e6f5538a6..b70b6d14f 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -25,8 +25,8 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" @@ -38,6 +38,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -91,6 +92,7 @@ var AllSources = [...]subscraping.Source{ &robtex.Source{}, &rsecloud.Source{}, &securitytrails.Source{}, + &profundis.Source{}, &shodan.Source{}, &sitedossier.Source{}, &threatbook.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 94c588160..9c2676b00 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -43,6 +43,7 @@ var ( "robtex", "rsecloud", "securitytrails", + "profundis", "shodan", "sitedossier", "threatbook", @@ -88,6 +89,7 @@ var ( // "riddler", // failing due to cloudfront protection "rsecloud", "securitytrails", + "profundis", "shodan", "windvane", "virustotal", diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go new file mode 100644 index 000000000..bb0bb8d7c --- /dev/null +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -0,0 +1,127 @@ +// Package profundis logic +package profundis + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "strings" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + requestBody, err := json.Marshal(map[string]string{"domain": domain}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + headers := map[string]string{ + "Content-Type": "application/json", + "X-API-KEY": randomApiKey, + "Accept": "text/event-stream", + } + + resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "", + headers, bytes.NewReader(requestBody)) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: line}: + s.results++ + } + } + + if err := scanner.Err(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + return results +} + +func (s *Source) Name() string { + return "profundis" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 037d0636037216d8082ddfe6b59fff431473570a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 8 Dec 2025 15:25:17 +0300 Subject: [PATCH 094/132] minor --- pkg/subscraping/sources/profundis/profundis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go index bb0bb8d7c..698b98ac0 100644 --- a/pkg/subscraping/sources/profundis/profundis.go +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -106,7 +106,7 @@ func (s *Source) IsDefault() bool { } func (s *Source) HasRecursiveSupport() bool { - return true + return false } func (s *Source) NeedsKey() bool { From 6b3df63b6694537f3119063380f5c3c815d6a5a7 Mon Sep 17 00:00:00 2001 From: knakul853 Date: Mon, 8 Dec 2025 19:32:53 +0530 Subject: [PATCH 095/132] fix: added unit test and tested e2e --- pkg/subscraping/sources/censys/censys.go | 96 +++++----- pkg/subscraping/sources/censys/censys_test.go | 170 ++++++++++++++++++ 2 files changed, 215 insertions(+), 51 deletions(-) create mode 100644 pkg/subscraping/sources/censys/censys_test.go diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index b5c17a8b4..6fc61c3ce 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -2,14 +2,14 @@ package censys import ( + "bytes" "context" - "strconv" + "net/http" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - urlutil "github.com/projectdiscovery/utils/url" ) const ( @@ -17,56 +17,42 @@ const ( maxPerPage = 100 ) +// Platform API request body +type searchRequest struct { + Query string `json:"query"` + Fields []string `json:"fields,omitempty"` + PageSize int `json:"page_size,omitempty"` + Cursor string `json:"cursor,omitempty"` +} + +// Platform API response structures type response struct { - Code int `json:"code"` - Status string `json:"status"` Result result `json:"result"` } type result struct { - Query string `json:"query"` - Total float64 `json:"total"` - DurationMS int `json:"duration_ms"` - Hits []hit `json:"hits"` - Links links `json:"links"` + Hits []hit `json:"hits"` + Cursor string `json:"cursor"` + Total int64 `json:"total"` } type hit struct { - Parsed parsed `json:"parsed"` - Names []string `json:"names"` - FingerprintSha256 string `json:"fingerprint_sha256"` + Certificate certificate `json:"certificate"` } -type parsed struct { - ValidityPeriod validityPeriod `json:"validity_period"` - SubjectDN string `json:"subject_dn"` - IssuerDN string `json:"issuer_dn"` -} - -type validityPeriod struct { - NotAfter string `json:"not_after"` - NotBefore string `json:"not_before"` -} - -type links struct { - Next string `json:"next"` - Prev string `json:"prev"` +type certificate struct { + Names []string `json:"names"` } // Source is the passive scraping agent type Source struct { - apiKeys []apiKey + apiKeys []string timeTaken time.Duration errors int results int skipped bool } -type apiKey struct { - token string - secret string -} - // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) @@ -80,41 +66,51 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey.token == "" || randomApiKey.secret == "" { + if randomApiKey == "" { s.skipped = true return } - certSearchEndpoint := "https://search.censys.io/api/v2/certificates/search" + searchEndpoint := "https://api.platform.censys.io/v3/global/search/query" cursor := "" currentPage := 1 + for { select { case <-ctx.Done(): return default: } - certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint) + + // Build request body + reqBody := searchRequest{ + Query: "certificate.names: " + domain, + Fields: []string{"certificate.names"}, + PageSize: maxPerPage, + } + if cursor != "" { + reqBody.Cursor = cursor + } + + bodyBytes, err := jsoniter.Marshal(reqBody) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } - certSearchEndpointUrl.Params.Add("q", domain) - certSearchEndpointUrl.Params.Add("per_page", strconv.Itoa(maxPerPage)) - if cursor != "" { - certSearchEndpointUrl.Params.Add("cursor", cursor) - } - + // Make POST request with Bearer token auth resp, err := session.HTTPRequest( ctx, - "GET", - certSearchEndpointUrl.String(), + http.MethodPost, + searchEndpoint, "", - nil, - nil, - subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret}, + map[string]string{ + "Content-Type": "application/json", + "Authorization": "Bearer " + randomApiKey, + }, + bytes.NewReader(bodyBytes), + subscraping.BasicAuth{}, ) if err != nil { @@ -136,7 +132,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, hit := range censysResponse.Result.Hits { - for _, name := range hit.Names { + for _, name := range hit.Certificate.Names { select { case <-ctx.Done(): return @@ -146,7 +142,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } } - cursor = censysResponse.Result.Links.Next + cursor = censysResponse.Result.Cursor if cursor == "" || currentPage >= maxCensysPages { break } @@ -175,9 +171,7 @@ func (s *Source) NeedsKey() bool { } func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - return apiKey{k, v} - }) + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { diff --git a/pkg/subscraping/sources/censys/censys_test.go b/pkg/subscraping/sources/censys/censys_test.go new file mode 100644 index 000000000..6bf9c6d1b --- /dev/null +++ b/pkg/subscraping/sources/censys/censys_test.go @@ -0,0 +1,170 @@ +package censys + +import ( + "context" + "io" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createTestMultiRateLimiter creates a MultiLimiter for testing +func createTestMultiRateLimiter(ctx context.Context) *ratelimit.MultiLimiter { + mrl, _ := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ + Key: "censys", + IsUnlimited: false, + MaxCount: math.MaxInt32, + Duration: time.Millisecond, + }) + return mrl +} + +func TestCensysSource_NoApiKey(t *testing.T) { + source := &Source{} + // Don't add any API keys + + ctx := context.Background() + multiRateLimiter := createTestMultiRateLimiter(ctx) + session := &subscraping.Session{ + Client: http.DefaultClient, + MultiRateLimiter: multiRateLimiter, + } + + ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, "censys") + results := source.Run(ctxWithValue, "example.com", session) + + // Collect all results + var resultCount int + for range results { + resultCount++ + } + + // Should be skipped when no API key + stats := source.Statistics() + assert.True(t, stats.Skipped, "expected source to be skipped without API key") + assert.Equal(t, 0, resultCount, "expected no results when skipped") +} + +func TestCensysSource_ContextCancellation(t *testing.T) { + // Create a server that delays response + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(500 * time.Millisecond) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"result": {"hits": [], "cursor": "", "total": 0}}`)) + })) + defer server.Close() + + source := &Source{} + source.AddApiKeys([]string{"test_pat"}) + + ctx := context.Background() + multiRateLimiter := createTestMultiRateLimiter(ctx) + session := &subscraping.Session{ + Client: server.Client(), + MultiRateLimiter: multiRateLimiter, + } + + // Create a context that will be cancelled + ctxCancellable, cancel := context.WithCancel(ctx) + ctxWithValue := context.WithValue(ctxCancellable, subscraping.CtxSourceArg, "censys") + + results := source.Run(ctxWithValue, "example.com", session) + + // Cancel immediately + cancel() + + // Should exit quickly without blocking + done := make(chan struct{}) + go func() { + for range results { + // drain + } + close(done) + }() + + select { + case <-done: + // Good - completed quickly + case <-time.After(2 * time.Second): + t.Fatal("context cancellation did not stop the source in time") + } +} + +func TestCensysSource_Metadata(t *testing.T) { + source := &Source{} + + assert.Equal(t, "censys", source.Name()) + assert.True(t, source.IsDefault()) + assert.False(t, source.HasRecursiveSupport()) + assert.True(t, source.NeedsKey()) +} + +func TestCensysSource_AddApiKeys(t *testing.T) { + source := &Source{} + + keys := []string{"pat_token_1", "pat_token_2"} + source.AddApiKeys(keys) + + require.Len(t, source.apiKeys, 2) + assert.Equal(t, "pat_token_1", source.apiKeys[0]) + assert.Equal(t, "pat_token_2", source.apiKeys[1]) +} + +func TestCensysSource_Statistics(t *testing.T) { + source := &Source{ + errors: 2, + results: 10, + timeTaken: 5 * time.Second, + skipped: false, + } + + stats := source.Statistics() + assert.Equal(t, 2, stats.Errors) + assert.Equal(t, 10, stats.Results) + assert.Equal(t, 5*time.Second, stats.TimeTaken) + assert.False(t, stats.Skipped) +} + +func TestCensysSource_RequestValidation(t *testing.T) { + // Create mock server to validate request format + var capturedRequest struct { + method string + authHeader string + contentType string + body string + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedRequest.method = r.Method + capturedRequest.authHeader = r.Header.Get("Authorization") + capturedRequest.contentType = r.Header.Get("Content-Type") + body, _ := io.ReadAll(r.Body) + capturedRequest.body = string(body) + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{ + "result": { + "hits": [{"certificate": {"names": ["sub.example.com"]}}], + "cursor": "", + "total": 1 + } + }`)) + })) + defer server.Close() + + // Note: This test validates request format expectations + // The actual source uses hardcoded URL, so this primarily tests expectations + + // Verify expected request format + assert.Equal(t, http.MethodPost, "POST", "Censys Platform API should use POST") + assert.True(t, strings.HasPrefix("Bearer test_token", "Bearer "), "Should use Bearer auth") + assert.Equal(t, "application/json", "application/json", "Should use JSON content type") +} From 8cf49e7e31ef14b18a2928cf0a64e3cb0f5b5c60 Mon Sep 17 00:00:00 2001 From: knakul853 Date: Mon, 8 Dec 2025 19:33:54 +0530 Subject: [PATCH 096/132] fix: added go doc --- pkg/subscraping/sources/censys/censys.go | 28 +++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 6fc61c3ce..85cc82eda 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -13,8 +13,20 @@ import ( ) const ( + // maxCensysPages is the maximum number of pages to fetch from the API maxCensysPages = 10 - maxPerPage = 100 + // maxPerPage is the maximum number of results per page + maxPerPage = 100 + // baseURL is the Censys Platform API base URL + baseURL = "https://api.platform.censys.io" + // searchEndpoint is the global data search query endpoint + searchEndpoint = "/v3/global/search/query" + // queryPrefix is the Censys query language prefix for certificate name search + queryPrefix = "certificate.names: " + // authHeaderPrefix is the Bearer token prefix for Authorization header + authHeaderPrefix = "Bearer " + // contentTypeJSON is the Content-Type header value for JSON + contentTypeJSON = "application/json" ) // Platform API request body @@ -65,13 +77,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + // PickRandom selects a random API key from configured keys. + // This enables load balancing when users configure multiple PATs + // (e.g., CENSYS_API_KEY=pat1,pat2,pat3) to distribute requests + // and avoid hitting rate limits on a single key. randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } - searchEndpoint := "https://api.platform.censys.io/v3/global/search/query" + apiURL := baseURL + searchEndpoint cursor := "" currentPage := 1 @@ -84,7 +100,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Build request body reqBody := searchRequest{ - Query: "certificate.names: " + domain, + Query: queryPrefix + domain, Fields: []string{"certificate.names"}, PageSize: maxPerPage, } @@ -103,11 +119,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp, err := session.HTTPRequest( ctx, http.MethodPost, - searchEndpoint, + apiURL, "", map[string]string{ - "Content-Type": "application/json", - "Authorization": "Bearer " + randomApiKey, + "Content-Type": contentTypeJSON, + "Authorization": authHeaderPrefix + randomApiKey, }, bytes.NewReader(bodyBytes), subscraping.BasicAuth{}, From b7193b46c3aa0a8d54961600b620c58f7c5a6562 Mon Sep 17 00:00:00 2001 From: knakul853 Date: Mon, 8 Dec 2025 20:04:27 +0530 Subject: [PATCH 097/132] fix: response strucutre --- pkg/subscraping/sources/censys/censys.go | 71 ++++++++++++----- pkg/subscraping/sources/censys/censys_test.go | 77 ++++++------------- 2 files changed, 72 insertions(+), 76 deletions(-) diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 85cc82eda..0008a78ab 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "net/http" + "strings" "time" jsoniter "github.com/json-iterator/go" @@ -22,13 +23,21 @@ const ( // searchEndpoint is the global data search query endpoint searchEndpoint = "/v3/global/search/query" // queryPrefix is the Censys query language prefix for certificate name search - queryPrefix = "certificate.names: " + queryPrefix = "cert.names: " // authHeaderPrefix is the Bearer token prefix for Authorization header authHeaderPrefix = "Bearer " // contentTypeJSON is the Content-Type header value for JSON contentTypeJSON = "application/json" + // orgIDHeader is the header name for organization ID + orgIDHeader = "X-Organization-ID" ) +// apiKey holds the Personal Access Token and optional Organization ID +type apiKey struct { + pat string + orgID string +} + // Platform API request body type searchRequest struct { Query string `json:"query"` @@ -43,22 +52,26 @@ type response struct { } type result struct { - Hits []hit `json:"hits"` - Cursor string `json:"cursor"` - Total int64 `json:"total"` + Hits []hit `json:"hits"` + TotalHits int64 `json:"total_hits"` + NextPageToken string `json:"next_page_token"` } type hit struct { - Certificate certificate `json:"certificate"` + CertificateV1 certificateV1 `json:"certificate_v1"` +} + +type certificateV1 struct { + Resource resource `json:"resource"` } -type certificate struct { +type resource struct { Names []string `json:"names"` } // Source is the passive scraping agent type Source struct { - apiKeys []string + apiKeys []apiKey timeTaken time.Duration errors int results int @@ -79,10 +92,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // PickRandom selects a random API key from configured keys. // This enables load balancing when users configure multiple PATs - // (e.g., CENSYS_API_KEY=pat1,pat2,pat3) to distribute requests + // (e.g., CENSYS_API_KEY=pat1:org1,pat2:org2) to distribute requests // and avoid hitting rate limits on a single key. randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { + if randomApiKey.pat == "" { s.skipped = true return } @@ -101,7 +114,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Build request body reqBody := searchRequest{ Query: queryPrefix + domain, - Fields: []string{"certificate.names"}, + Fields: []string{"cert.names"}, PageSize: maxPerPage, } if cursor != "" { @@ -115,16 +128,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // Make POST request with Bearer token auth + // Build headers with Bearer token auth + headers := map[string]string{ + "Content-Type": contentTypeJSON, + "Authorization": authHeaderPrefix + randomApiKey.pat, + } + // Add Organization ID header if provided + if randomApiKey.orgID != "" { + headers[orgIDHeader] = randomApiKey.orgID + } + + // Make POST request resp, err := session.HTTPRequest( ctx, http.MethodPost, apiURL, "", - map[string]string{ - "Content-Type": contentTypeJSON, - "Authorization": authHeaderPrefix + randomApiKey, - }, + headers, bytes.NewReader(bodyBytes), subscraping.BasicAuth{}, ) @@ -138,17 +158,15 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var censysResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) + resp.Body.Close() if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - session.DiscardHTTPResponse(resp) return } - session.DiscardHTTPResponse(resp) - for _, hit := range censysResponse.Result.Hits { - for _, name := range hit.Certificate.Names { + for _, name := range hit.CertificateV1.Resource.Names { select { case <-ctx.Done(): return @@ -158,7 +176,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } } - cursor = censysResponse.Result.Cursor + cursor = censysResponse.Result.NextPageToken if cursor == "" || currentPage >= maxCensysPages { break } @@ -186,8 +204,19 @@ func (s *Source) NeedsKey() bool { return true } +// AddApiKeys parses and adds API keys. +// Format: "PAT:ORG_ID" where ORG_ID is required for paid accounts. +// Example: "censys_xxx_token:12345678-91011-1213" func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys + s.apiKeys = subscraping.CreateApiKeys(keys, func(pat, orgID string) apiKey { + return apiKey{pat: pat, orgID: orgID} + }) + // Also support single PAT without org ID for free users + for _, key := range keys { + if !strings.Contains(key, ":") && key != "" { + s.apiKeys = append(s.apiKeys, apiKey{pat: key, orgID: ""}) + } + } } func (s *Source) Statistics() subscraping.Statistics { diff --git a/pkg/subscraping/sources/censys/censys_test.go b/pkg/subscraping/sources/censys/censys_test.go index 6bf9c6d1b..f928698f0 100644 --- a/pkg/subscraping/sources/censys/censys_test.go +++ b/pkg/subscraping/sources/censys/censys_test.go @@ -2,11 +2,8 @@ package censys import ( "context" - "io" "math" "net/http" - "net/http/httptest" - "strings" "testing" "time" @@ -54,21 +51,14 @@ func TestCensysSource_NoApiKey(t *testing.T) { } func TestCensysSource_ContextCancellation(t *testing.T) { - // Create a server that delays response - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(500 * time.Millisecond) - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"result": {"hits": [], "cursor": "", "total": 0}}`)) - })) - defer server.Close() - source := &Source{} - source.AddApiKeys([]string{"test_pat"}) + // Add a key with PAT:ORG_ID format + source.AddApiKeys([]string{"test_pat:test_org_id"}) ctx := context.Background() multiRateLimiter := createTestMultiRateLimiter(ctx) session := &subscraping.Session{ - Client: server.Client(), + Client: http.DefaultClient, MultiRateLimiter: multiRateLimiter, } @@ -108,14 +98,27 @@ func TestCensysSource_Metadata(t *testing.T) { } func TestCensysSource_AddApiKeys(t *testing.T) { - source := &Source{} + t.Run("PAT with OrgID", func(t *testing.T) { + source := &Source{} + keys := []string{"pat_token_1:org_id_1", "pat_token_2:org_id_2"} + source.AddApiKeys(keys) + + require.Len(t, source.apiKeys, 2) + assert.Equal(t, "pat_token_1", source.apiKeys[0].pat) + assert.Equal(t, "org_id_1", source.apiKeys[0].orgID) + assert.Equal(t, "pat_token_2", source.apiKeys[1].pat) + assert.Equal(t, "org_id_2", source.apiKeys[1].orgID) + }) - keys := []string{"pat_token_1", "pat_token_2"} - source.AddApiKeys(keys) + t.Run("PAT without OrgID (free user)", func(t *testing.T) { + source := &Source{} + keys := []string{"pat_token_only"} + source.AddApiKeys(keys) - require.Len(t, source.apiKeys, 2) - assert.Equal(t, "pat_token_1", source.apiKeys[0]) - assert.Equal(t, "pat_token_2", source.apiKeys[1]) + require.Len(t, source.apiKeys, 1) + assert.Equal(t, "pat_token_only", source.apiKeys[0].pat) + assert.Equal(t, "", source.apiKeys[0].orgID) + }) } func TestCensysSource_Statistics(t *testing.T) { @@ -132,39 +135,3 @@ func TestCensysSource_Statistics(t *testing.T) { assert.Equal(t, 5*time.Second, stats.TimeTaken) assert.False(t, stats.Skipped) } - -func TestCensysSource_RequestValidation(t *testing.T) { - // Create mock server to validate request format - var capturedRequest struct { - method string - authHeader string - contentType string - body string - } - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - capturedRequest.method = r.Method - capturedRequest.authHeader = r.Header.Get("Authorization") - capturedRequest.contentType = r.Header.Get("Content-Type") - body, _ := io.ReadAll(r.Body) - capturedRequest.body = string(body) - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{ - "result": { - "hits": [{"certificate": {"names": ["sub.example.com"]}}], - "cursor": "", - "total": 1 - } - }`)) - })) - defer server.Close() - - // Note: This test validates request format expectations - // The actual source uses hardcoded URL, so this primarily tests expectations - - // Verify expected request format - assert.Equal(t, http.MethodPost, "POST", "Censys Platform API should use POST") - assert.True(t, strings.HasPrefix("Bearer test_token", "Bearer "), "Should use Bearer auth") - assert.Equal(t, "application/json", "application/json", "Should use JSON content type") -} From 0b6432d8cc670605ee03f89e12a8bf8626c19934 Mon Sep 17 00:00:00 2001 From: knakul853 Date: Mon, 8 Dec 2025 20:09:59 +0530 Subject: [PATCH 098/132] fix: linter errors --- pkg/subscraping/sources/censys/censys.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 0008a78ab..388cc301c 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -111,7 +111,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } - // Build request body reqBody := searchRequest{ Query: queryPrefix + domain, Fields: []string{"cert.names"}, @@ -128,7 +127,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // Build headers with Bearer token auth headers := map[string]string{ "Content-Type": contentTypeJSON, "Authorization": authHeaderPrefix + randomApiKey.pat, @@ -138,7 +136,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers[orgIDHeader] = randomApiKey.orgID } - // Make POST request resp, err := session.HTTPRequest( ctx, http.MethodPost, @@ -158,7 +155,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var censysResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) - resp.Body.Close() + _ = resp.Body.Close() if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ From e7738aa6bda8159e2ebcb4b108320fe51e9f1612 Mon Sep 17 00:00:00 2001 From: nohehf Date: Tue, 16 Dec 2025 16:35:04 +0100 Subject: [PATCH 099/132] feat: merklemap source --- pkg/passive/sources.go | 4 +- .../sources/merklemap/merklemap.go | 162 ++++++++++++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 pkg/subscraping/sources/merklemap/merklemap.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index e6f5538a6..86da44142 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -25,8 +25,8 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" @@ -36,6 +36,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" @@ -81,6 +82,7 @@ var AllSources = [...]subscraping.Source{ &hackertarget.Source{}, &intelx.Source{}, &netlas.Source{}, + &merklemap.Source{}, &onyphe.Source{}, &leakix.Source{}, &quake.Source{}, diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go new file mode 100644 index 000000000..b6cb2d8d2 --- /dev/null +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -0,0 +1,162 @@ +// Package merklemap logic +package merklemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + // Default headers + headers := map[string]string{ + "accept": "application/json", + // Set a user agent to prevent random one from pkg/subscraping/agent.go, it triggers the cloudflare protection of the api + "User-Agent": "subfinder", + } + // Pick an API key + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["Authorization"] = "Bearer " + randomApiKey + } + + // Fetch all pages with pagination + // https://www.merklemap.com/documentation/search + s.fetchAllPages(ctx, domain, headers, session, results) + }() + return results +} + +// fetchAllPages fetches all pages of results using pagination +func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + baseURL := "https://api.merklemap.com/v1/search?query=" + "*." + domain + totalCount := math.MaxInt + processedResults := 0 + + // Iterate through all pages + for page := 0; processedResults < totalCount; page++ { + pageResp, err := s.fetchPage(ctx, baseURL, page, headers, session) + + if page == 0 { + totalCount = pageResp.Count + } + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + // Stop if this page returned no results + if len(pageResp.Results) == 0 { + break + } + + for _, result := range pageResp.Results { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: result.Hostname, + } + s.results++ + processedResults++ + } + + } +} + +// fetchPage fetches a single page of results +func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) { + url := baseURL + "&page=" + strconv.Itoa(page) + + resp, err := session.Get(ctx, url, "", headers) + if err != nil { + return nil, err + } + defer session.DiscardHTTPResponse(resp) + + if resp.StatusCode != 200 { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, err) + } + return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, string(respBody)) + } + + var pageResponse response + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(bytes.NewReader(respBody)) + if err := decoder.Decode(&pageResponse); err != nil { + return nil, err + } + + return &pageResponse, nil +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "merklemap" +} + +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} + +type response struct { + Count int `json:"count"` + Results []struct { + Hostname string `json:"hostname"` + SubjectCommonName string `json:"subject_common_name"` + FirstSeen string `json:"first_seen"` + } `json:"results"` +} From 7ac8b6cfcc048bb6017b65b068744659db89f077 Mon Sep 17 00:00:00 2001 From: nohehf Date: Tue, 16 Dec 2025 17:55:13 +0100 Subject: [PATCH 100/132] feat: skip merklemap if no api key --- pkg/subscraping/sources/merklemap/merklemap.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go index b6cb2d8d2..17fa82d76 100644 --- a/pkg/subscraping/sources/merklemap/merklemap.go +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -34,16 +34,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.timeTaken = time.Since(startTime) close(results) }(time.Now()) + // Pick an API key, skip if no key is found + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + // Default headers headers := map[string]string{ "accept": "application/json", // Set a user agent to prevent random one from pkg/subscraping/agent.go, it triggers the cloudflare protection of the api - "User-Agent": "subfinder", - } - // Pick an API key - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey != "" { - headers["Authorization"] = "Bearer " + randomApiKey + "User-Agent": "subfinder", + "Authorization": "Bearer " + randomApiKey, } // Fetch all pages with pagination From e18ed93e52ba8077c365043f635f1974f6a09295 Mon Sep 17 00:00:00 2001 From: nohehf Date: Tue, 16 Dec 2025 18:03:34 +0100 Subject: [PATCH 101/132] fix: implement feedback --- pkg/subscraping/sources/merklemap/merklemap.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go index 17fa82d76..aa4881e72 100644 --- a/pkg/subscraping/sources/merklemap/merklemap.go +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "math" + "net/url" "strconv" "time" @@ -58,7 +59,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // fetchAllPages fetches all pages of results using pagination func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { - baseURL := "https://api.merklemap.com/v1/search?query=" + "*." + domain + baseURL := "https://api.merklemap.com/v1/search?query=" + url.QueryEscape("*."+domain) totalCount := math.MaxInt processedResults := 0 @@ -66,16 +67,16 @@ func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[s for page := 0; processedResults < totalCount; page++ { pageResp, err := s.fetchPage(ctx, baseURL, page, headers, session) - if page == 0 { - totalCount = pageResp.Count - } - if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } + if page == 0 { + totalCount = pageResp.Count + } + // Stop if this page returned no results if len(pageResp.Results) == 0 { break From 692e30a592c19dfe2f82fc4201ff266bd9f6ce2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 18 Dec 2025 12:24:51 +0300 Subject: [PATCH 102/132] add thc --- pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 2 + pkg/subscraping/sources/thc/thc.go | 127 +++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 pkg/subscraping/sources/thc/thc.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index b70b6d14f..64f4dd8e7 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -48,6 +48,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" @@ -108,6 +109,7 @@ var AllSources = [...]subscraping.Source{ &builtwith.Source{}, &hudsonrock.Source{}, &digitalyama.Source{}, + &thc.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 9c2676b00..a1f99be8d 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -60,6 +60,7 @@ var ( "builtwith", "hudsonrock", "digitalyama", + "thc", } expectedDefaultSources = []string{ @@ -100,6 +101,7 @@ var ( // "reconcloud", "builtwith", "digitalyama", + "thc", } expectedDefaultRecursiveSources = []string{ diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go new file mode 100644 index 000000000..238062193 --- /dev/null +++ b/pkg/subscraping/sources/thc/thc.go @@ -0,0 +1,127 @@ +// Package thc logic +package thc + +import ( + "bytes" + "context" + "encoding/json" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Domains []struct { + Domain string `json:"domain"` + } `json:"domains"` + NextPageState string `json:"next_page_state"` +} + +// Source is the passive scraping agent +type Source struct { + timeTaken time.Duration + errors int + results int + skipped bool +} + +type requestBody struct { + Domain string `json:"domain"` + PageState string `json:"page_state"` + Limit int `json:"limit"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + var pageState string + headers := map[string]string{"Content-Type": "application/json"} + apiURL := "https://ip.thc.org/api/v1/lookup/subdomains" + + for { + reqBody := requestBody{ + Domain: domain, + PageState: pageState, + Limit: 1000, + } + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var thcResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&thcResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, domainRecord := range thcResponse.Domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domainRecord.Domain} + s.results++ + } + + pageState = thcResponse.NextPageState + + if pageState == "" { + break + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "thc" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // No API keys needed for THC +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 79a426007b2b2fdae5a2fc77d8bdf46aa18d5031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 18 Dec 2025 15:28:48 +0300 Subject: [PATCH 103/132] bump version --- pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index c7768520f..a91bbe839 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.10.1` +const version = `v2.11.0` // showBanner is used to show the banner to the user func showBanner() { From e2d7ecfd14619260c9cb6e6034a79f14fe18122b Mon Sep 17 00:00:00 2001 From: nohehf Date: Fri, 19 Dec 2025 17:06:04 +0100 Subject: [PATCH 104/132] fix: tests --- pkg/passive/sources_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 94c588160..7da6f1c4f 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -59,6 +59,7 @@ var ( "builtwith", "hudsonrock", "digitalyama", + "merklemap", } expectedDefaultSources = []string{ @@ -113,6 +114,7 @@ var ( "virustotal", "leakix", "facebook", + "merklemap", // "reconcloud", } ) From fe41f4904034feffded209ce93de34d3b6ce5002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sun, 4 Jan 2026 15:56:45 +0300 Subject: [PATCH 105/132] add reconeer --- pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 2 + pkg/subscraping/sources/reconeer/reconeer.go | 110 +++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 pkg/subscraping/sources/reconeer/reconeer.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 1e2683c7b..292d60656 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -43,6 +43,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" @@ -108,6 +109,7 @@ var AllSources = [...]subscraping.Source{ &facebook.Source{}, // &threatminer.Source{}, // failing api // &reconcloud.Source{}, // failing due to cloudflare bot protection + &reconeer.Source{}, &builtwith.Source{}, &hudsonrock.Source{}, &digitalyama.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 0a5ff9602..82aa0336b 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -57,6 +57,7 @@ var ( "facebook", // "threatminer", // "reconcloud", + "reconeer", "builtwith", "hudsonrock", "digitalyama", @@ -100,6 +101,7 @@ var ( "facebook", // "threatminer", // "reconcloud", + "reconeer", "builtwith", "digitalyama", "thc", diff --git a/pkg/subscraping/sources/reconeer/reconeer.go b/pkg/subscraping/sources/reconeer/reconeer.go new file mode 100644 index 000000000..6463da9a5 --- /dev/null +++ b/pkg/subscraping/sources/reconeer/reconeer.go @@ -0,0 +1,110 @@ +package reconeer + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Subdomains []subdomain `json:"subdomains"` +} + +type subdomain struct { + Subdomain string `json:"subdomain"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + headers := map[string]string{ + "Accept": "application/json", + "X-API-KEY": randomApiKey, + } + apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) + resp, err := session.Get(ctx, apiURL, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + defer session.DiscardHTTPResponse(resp) + + if resp.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} + s.errors++ + return + } + var responseData response + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&responseData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + for _, result := range responseData.Subdomains { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: + s.results++ + } + } + }() + return results +} + +func (s *Source) Name() string { + return "reconeer" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 8eacfb143a84eb9a8da5b523f1cc0c0c3428ebb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 5 Jan 2026 11:42:17 +0300 Subject: [PATCH 106/132] bump version --- pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index a91bbe839..4d7a9532c 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.11.0` +const version = `v2.12.0` // showBanner is used to show the banner to the user func showBanner() { From e96320ec383b556de8ea6c7d3fb261ad2a4bc0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 5 Jan 2026 14:14:20 +0300 Subject: [PATCH 107/132] add request tracking to -stats flag Adds request counting to all sources to help users monitor API usage and debug quota consumption issues like #1562. Changes: - Add Requests field to Statistics struct - Track HTTP requests in all 53 sources - Display request count in stats output Closes #1698 --- pkg/runner/stats.go | 4 ++-- pkg/subscraping/sources/alienvault/alienvault.go | 4 ++++ pkg/subscraping/sources/anubis/anubis.go | 4 ++++ pkg/subscraping/sources/bevigil/bevigil.go | 4 ++++ pkg/subscraping/sources/bufferover/bufferover.go | 4 ++++ pkg/subscraping/sources/builtwith/builtwith.go | 4 ++++ pkg/subscraping/sources/c99/c99.go | 4 ++++ pkg/subscraping/sources/censys/censys.go | 4 ++++ pkg/subscraping/sources/certspotter/certspotter.go | 5 +++++ pkg/subscraping/sources/chaos/chaos.go | 4 ++++ pkg/subscraping/sources/chinaz/chinaz.go | 4 ++++ pkg/subscraping/sources/commoncrawl/commoncrawl.go | 5 +++++ pkg/subscraping/sources/crtsh/crtsh.go | 4 ++++ pkg/subscraping/sources/digitalyama/digitalyama.go | 4 ++++ pkg/subscraping/sources/digitorus/digitorus.go | 4 ++++ pkg/subscraping/sources/dnsdb/dnsdb.go | 5 +++++ pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 4 ++++ pkg/subscraping/sources/dnsrepo/dnsrepo.go | 4 ++++ pkg/subscraping/sources/domainsproject/domainsproject.go | 4 ++++ pkg/subscraping/sources/driftnet/driftnet.go | 4 ++++ pkg/subscraping/sources/facebook/ctlogs.go | 4 ++++ pkg/subscraping/sources/fofa/fofa.go | 4 ++++ pkg/subscraping/sources/fullhunt/fullhunt.go | 4 ++++ pkg/subscraping/sources/github/github.go | 5 +++++ pkg/subscraping/sources/gitlab/gitlab.go | 5 +++++ pkg/subscraping/sources/hackertarget/hackertarget.go | 4 ++++ pkg/subscraping/sources/hudsonrock/hudsonrock.go | 4 ++++ pkg/subscraping/sources/intelx/intelx.go | 5 +++++ pkg/subscraping/sources/leakix/leakix.go | 4 ++++ pkg/subscraping/sources/merklemap/merklemap.go | 4 ++++ pkg/subscraping/sources/netlas/netlas.go | 5 +++++ pkg/subscraping/sources/onyphe/onyphe.go | 4 ++++ pkg/subscraping/sources/profundis/profundis.go | 4 ++++ pkg/subscraping/sources/pugrecon/pugrecon.go | 4 ++++ pkg/subscraping/sources/quake/quake.go | 4 ++++ pkg/subscraping/sources/rapiddns/rapiddns.go | 4 ++++ pkg/subscraping/sources/reconcloud/reconcloud.go | 4 ++++ pkg/subscraping/sources/reconeer/reconeer.go | 4 ++++ pkg/subscraping/sources/redhuntlabs/redhuntlabs.go | 5 +++++ pkg/subscraping/sources/riddler/riddler.go | 4 ++++ pkg/subscraping/sources/rsecloud/rsecloud.go | 4 ++++ pkg/subscraping/sources/securitytrails/securitytrails.go | 6 ++++++ pkg/subscraping/sources/shodan/shodan.go | 4 ++++ pkg/subscraping/sources/sitedossier/sitedossier.go | 4 ++++ pkg/subscraping/sources/thc/thc.go | 4 ++++ pkg/subscraping/sources/threatbook/threatbook.go | 4 ++++ pkg/subscraping/sources/threatcrowd/threatcrowd.go | 4 ++++ pkg/subscraping/sources/threatminer/threatminer.go | 4 ++++ pkg/subscraping/sources/virustotal/virustotal.go | 4 ++++ pkg/subscraping/sources/waybackarchive/waybackarchive.go | 4 ++++ pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go | 4 ++++ pkg/subscraping/sources/windvane/windvane.go | 4 ++++ pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go | 4 ++++ pkg/subscraping/types.go | 1 + 54 files changed, 221 insertions(+), 2 deletions(-) diff --git a/pkg/runner/stats.go b/pkg/runner/stats.go index d34b20f82..ccaa13e32 100644 --- a/pkg/runner/stats.go +++ b/pkg/runner/stats.go @@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) { if sourceStats.Skipped { skipped = append(skipped, fmt.Sprintf(" %s", source)) } else { - lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors)) + lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors)) } } if len(lines) > 0 { - gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56)) + gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68)) gologger.Print().Msg(strings.Join(lines, "\n")) gologger.Print().Msgf("\n") } diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go index 6af4268f2..a67727946 100644 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration results int errors int + requests int apiKeys []string skipped bool } @@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "", map[string]string{"Authorization": "Bearer " + randomApiKey}) if err != nil && resp == nil { @@ -110,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go index 96d21e784..c2328aa96 100644 --- a/pkg/subscraping/sources/anubis/anubis.go +++ b/pkg/subscraping/sources/anubis/anubis.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -94,6 +97,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go index cbdcf0f43..6306f1216 100644 --- a/pkg/subscraping/sources/bevigil/bevigil.go +++ b/pkg/subscraping/sources/bevigil/bevigil.go @@ -21,6 +21,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) + s.requests++ resp, err := session.Get(ctx, getUrl, "", map[string]string{ "X-Access-Token": randomApiKey, "User-Agent": "subfinder", }) @@ -106,6 +109,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 198d31e0b..2fa3358d8 100644 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { + s.requests++ resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) if err != nil && resp == nil { @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go index a22043375..155a06fa0 100644 --- a/pkg/subscraping/sources/builtwith/builtwith.go +++ b/pkg/subscraping/sources/builtwith/builtwith.go @@ -35,6 +35,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -112,6 +115,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go index 124686d27..a90405897 100644 --- a/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) @@ -119,6 +122,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 388cc301c..4b015dd6c 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -75,6 +75,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -83,6 +84,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -136,6 +138,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers[orgIDHeader] = randomApiKey.orgID } + s.requests++ resp, err := session.HTTPRequest( ctx, http.MethodPost, @@ -220,6 +223,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index 35118ff65..e27a1cbf5 100644 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Authorization": "Bearer " + randomApiKey} cookies := "" + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -88,6 +91,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) + s.requests++ resp, err := session.Get(ctx, reqURL, cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -152,6 +156,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go index 489e3173a..8b7c1c99f 100644 --- a/pkg/subscraping/sources/chaos/chaos.go +++ b/pkg/subscraping/sources/chaos/chaos.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) } chaosClient := chaos.New(randomApiKey) + s.requests++ for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ Domain: domain, }) { @@ -86,6 +89,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go index 2fd6c9546..20e6206d6 100644 --- a/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -103,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index b1dda4910..b2ce3feb6 100644 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, indexURL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -117,6 +120,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } @@ -128,6 +132,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se return false default: var headers = map[string]string{"Host": "index.commoncrawl.org"} + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index caea79c42..552756c4b 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -145,6 +147,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -212,6 +215,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go index 8a89e686e..54a158c9b 100644 --- a/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -34,6 +35,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -48,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) + s.requests++ resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go index 91b3ca5b3..ec51b867d 100644 --- a/pkg/subscraping/sources/digitorus/digitorus.go +++ b/pkg/subscraping/sources/digitorus/digitorus.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -32,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound { @@ -99,6 +102,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go index 810d8a05b..b7464966c 100644 --- a/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results uint64 + requests int skipped bool } @@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -70,6 +72,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "Accept": "application/x-ndjson", } + s.requests++ offsetMax, err := getMaxOffset(ctx, session, headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -92,6 +95,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } url := urlTemplate + queryParams.Encode() + s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -195,6 +199,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: int(s.results), + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 8062a4954..f4c84b6d2 100644 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -25,6 +25,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -103,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go index f7faaaac0..cb8c34fa8 100644 --- a/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se token := randomApiInfo[0] apiKey := randomApiInfo[1] + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -112,6 +115,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go index 800727d78..8f4fd2bd3 100644 --- a/pkg/subscraping/sources/domainsproject/domainsproject.go +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain) + s.requests++ resp, err := session.HTTPRequest( ctx, "GET", @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index fa0d0bcdd..2a9b450a3 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors atomic.Int32 results atomic.Int32 + requests atomic.Int32 skipped bool } @@ -65,6 +66,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors.Store(0) s.results.Store(0) + s.requests.Store(0) // Waitgroup for subsources var wg sync.WaitGroup @@ -119,6 +121,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: int(s.errors.Load()), Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } @@ -139,6 +142,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc // Request requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) + s.requests.Add(1) resp, err := session.Get(ctx, requestURL, "", headers) if err != nil { // HTTP 204 is not an error from the Driftnet API diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index f0f37a4e0..ba6e9250e 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -78,6 +78,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -86,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 if len(s.apiKeys) == 0 { s.skipped = true @@ -108,6 +110,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err := session.Get(ctx, domainsURL, "", nil) if err != nil { s.errors++ @@ -192,6 +195,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go index b885476aa..9332e3ecc 100644 --- a/pkg/subscraping/sources/fofa/fofa.go +++ b/pkg/subscraping/sources/fofa/fofa.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // fofa api doc https://fofa.info/static_pages/api_help qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go index c53206e11..5fc10a01e 100644 --- a/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -98,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go index 69f5e151c..63599cd16 100644 --- a/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -86,6 +88,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } // Initial request to GitHub search + s.requests++ resp, err := session.Get(ctx, searchURL, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil && !isForbidden { @@ -167,6 +170,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r default: } + s.requests++ resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { @@ -277,6 +281,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go index e2a13a111..6325aad30 100644 --- a/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -37,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -66,6 +68,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * default: } + s.requests++ resp, err := session.Get(ctx, searchURL, "", headers) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -91,6 +94,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * go func(item item) { // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) + s.requests++ resp, err := session.Get(ctx, fileUrl, "", headers) if err != nil { if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { @@ -173,6 +177,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index f85b4fe88..2739a9a94 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + s.requests++ resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -109,5 +112,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go index b3e2e0ec8..d27ca4d31 100644 --- a/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -26,6 +26,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -99,6 +102,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go index 82ecaeb16..1e26b1f94 100644 --- a/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -56,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -85,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -112,6 +115,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err = session.Get(ctx, resultsURL, "", nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -179,6 +183,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go index 0d3526ab7..91f27d00c 100644 --- a/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers["api-key"] = randomApiKey } // Request + s.requests++ resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -100,6 +103,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go index aa4881e72..7e9147db4 100644 --- a/pkg/subscraping/sources/merklemap/merklemap.go +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -21,6 +21,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -97,6 +99,7 @@ func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[s func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) { url := baseURL + "&page=" + strconv.Itoa(page) + s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { return nil, err @@ -153,6 +156,7 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go index a0c8b86e7..8ac65fee3 100644 --- a/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -40,6 +40,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -47,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,6 +65,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + s.requests++ resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, @@ -120,6 +123,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) + s.requests++ resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, @@ -196,6 +200,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/onyphe/onyphe.go b/pkg/subscraping/sources/onyphe/onyphe.go index a304ebdb3..f1f880a6c 100644 --- a/pkg/subscraping/sources/onyphe/onyphe.go +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -36,6 +36,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -44,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -73,6 +75,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", url.QueryEscape("category:resolver domain:"+domain), page, pageSize) + s.requests++ resp, err = session.Get(ctx, urlWithQuery, "", headers) if err != nil { @@ -159,6 +162,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go index 698b98ac0..3bfd8e622 100644 --- a/pkg/subscraping/sources/profundis/profundis.go +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -52,6 +54,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "Accept": "text/event-stream", } + s.requests++ resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "", headers, bytes.NewReader(requestBody)) @@ -123,5 +126,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go index 855154153..4a0c7e539 100644 --- a/pkg/subscraping/sources/pugrecon/pugrecon.go +++ b/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -71,6 +73,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } apiURL := "https://pugrecon.com/api/v1/domains" + s.requests++ resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -150,5 +153,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go index 2e99fb87f..3dd7f8d72 100644 --- a/pkg/subscraping/sources/quake/quake.go +++ b/pkg/subscraping/sources/quake/quake.go @@ -36,6 +36,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -44,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -69,6 +71,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + s.requests++ resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, }, bytes.NewReader(requestBody)) @@ -153,5 +156,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go index eb7969795..d204a1b3f 100644 --- a/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -19,6 +19,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -41,6 +43,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -115,5 +118,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go index 61bb593da..67bf651cf 100644 --- a/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -102,5 +105,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconeer/reconeer.go b/pkg/subscraping/sources/reconeer/reconeer.go index 6463da9a5..9e4bae633 100644 --- a/pkg/subscraping/sources/reconeer/reconeer.go +++ b/pkg/subscraping/sources/reconeer/reconeer.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "X-API-KEY": randomApiKey, } apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) + s.requests++ resp, err := session.Get(ctx, apiURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -106,5 +109,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index 009d5d000..21f8420e9 100644 --- a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 pageSize := 1000 go func() { defer func(startTime time.Time) { @@ -56,6 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1] requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"} getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -82,6 +85,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -150,5 +154,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go index 7855a344b..a74ee7d60 100644 --- a/pkg/subscraping/sources/riddler/riddler.go +++ b/pkg/subscraping/sources/riddler/riddler.go @@ -15,6 +15,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -22,6 +23,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -29,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -91,5 +94,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go index 43d270b05..b2fc70da3 100644 --- a/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -24,6 +24,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -122,5 +125,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index 0e6846e55..c25700dfe 100644 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -31,6 +31,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -66,13 +68,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if scrollId == "" { var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) + s.requests++ resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) } if err != nil && ptr.Safe(resp).StatusCode == 403 { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) } @@ -156,5 +161,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index b2195e393..7213602e0 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) @@ -127,6 +130,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index dde639c40..b96315c36 100644 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba default: } + s.requests++ resp, err := session.SimpleGet(ctx, baseURL) isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound if err != nil && !isnotfound { @@ -110,5 +113,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go index 238062193..f8ebb9795 100644 --- a/pkg/subscraping/sources/thc/thc.go +++ b/pkg/subscraping/sources/thc/thc.go @@ -24,6 +24,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,6 +65,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -123,5 +126,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go index 6a6645d3b..a788ad53b 100644 --- a/pkg/subscraping/sources/threatbook/threatbook.go +++ b/pkg/subscraping/sources/threatbook/threatbook.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -51,6 +53,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -127,5 +130,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index 831659f61..cde72f99f 100644 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run queries the ThreatCrowd API for the given domain and returns found subdomains. @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func(startTime time.Time) { defer func() { @@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Client.Do(req) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -122,5 +125,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 77f5a586e..3d25f3362 100644 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -36,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -93,5 +96,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 446c7c309..812f40e20 100644 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -60,6 +62,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } + s.requests++ resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -125,6 +128,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index ecbe4d943..6f48f0583 100644 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -97,5 +100,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index c4efbdd2e..547c32945 100644 --- a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -33,6 +33,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -41,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -54,6 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -113,5 +116,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go index 92f972c40..b0a132862 100644 --- a/pkg/subscraping/sources/windvane/windvane.go +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -38,6 +38,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -45,6 +46,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -72,6 +74,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var err error requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}}) + s.requests++ resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", "", headers, bytes.NewReader(requestBody)) @@ -150,5 +153,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 70d491c60..6b10fa34f 100644 --- a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -69,6 +71,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) + s.requests++ resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil { @@ -132,5 +135,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 454ad776f..b5fc6c091 100644 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -28,6 +28,7 @@ type BasicAuth struct { // Statistics contains statistics about the scraping process type Statistics struct { TimeTaken time.Duration + Requests int Errors int Results int Skipped bool From 9b4705252e5fac968e912ec8c8615b9abda5b4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 5 Jan 2026 14:23:20 +0300 Subject: [PATCH 108/132] fix race condition in github and gitlab sources use atomic.Int32 for counter fields to prevent data races when multiple goroutines concurrently increment requests/results/errors --- pkg/subscraping/sources/github/github.go | 35 ++++++++++++------------ pkg/subscraping/sources/gitlab/gitlab.go | 33 +++++++++++----------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go index 63599cd16..609302851 100644 --- a/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -41,18 +42,18 @@ type response struct { type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int - requests int + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - s.requests = 0 + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) go func() { defer func(startTime time.Time) { @@ -88,12 +89,12 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } // Initial request to GitHub search - s.requests++ + s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil && !isForbidden { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -114,7 +115,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -124,7 +125,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -141,7 +142,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) @@ -170,7 +171,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r default: } - s.requests++ + s.requests.Add(1) resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { @@ -199,7 +200,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r session.DiscardHTTPResponse(resp) return case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: - s.results++ + s.results.Add(1) } } } @@ -217,7 +218,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r case <-ctx.Done(): return case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: - s.results++ + s.results.Add(1) } } } @@ -279,9 +280,9 @@ func (s *Source) AddApiKeys(keys []string) { func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - Requests: s.requests, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go index 6325aad30..5f101af9f 100644 --- a/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -20,9 +21,9 @@ import ( type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int - requests int + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 skipped bool } @@ -36,9 +37,9 @@ type item struct { // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - s.requests = 0 + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) go func() { defer func(startTime time.Time) { @@ -68,11 +69,11 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * default: } - s.requests++ + s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -83,7 +84,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = jsoniter.NewDecoder(resp.Body).Decode(&items) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -94,14 +95,14 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * go func(item item) { // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) - s.requests++ + s.requests.Add(1) resp, err := session.Get(ctx, fileUrl, "", headers) if err != nil { if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { session.DiscardHTTPResponse(resp) results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } } @@ -115,7 +116,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } for _, subdomain := range domainRegexp.FindAllString(line, -1) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + s.results.Add(1) } } session.DiscardHTTPResponse(resp) @@ -135,7 +136,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -175,9 +176,9 @@ func (s *Source) AddApiKeys(keys []string) { // Statistics returns the statistics for the source func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - Requests: s.requests, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } From 1ca2a67f929be09d36aa420704e5bb71723a98af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 5 Jan 2026 15:00:59 +0300 Subject: [PATCH 109/132] add optional API key support for sources - add KeyRequirement enum with three states: NoKey, OptionalKey, RequiredKey - add KeyRequirement() method to Source interface - update config generation to include optional sources - update env var loading for optional sources - update source listing with ~ marker for optional sources - mark hackertarget, leakix, reconeer as OptionalKey - fix hackertarget and reconeer to work without API key closes #1695 --- pkg/passive/sources.go | 4 ++-- pkg/runner/config.go | 4 +++- pkg/runner/options.go | 15 ++++++++++----- .../sources/alienvault/alienvault.go | 6 +++++- pkg/subscraping/sources/anubis/anubis.go | 6 +++++- pkg/subscraping/sources/bevigil/bevigil.go | 6 +++++- .../sources/bufferover/bufferover.go | 6 +++++- pkg/subscraping/sources/builtwith/builtwith.go | 6 +++++- pkg/subscraping/sources/c99/c99.go | 6 +++++- pkg/subscraping/sources/censys/censys.go | 6 +++++- .../sources/certspotter/certspotter.go | 6 +++++- pkg/subscraping/sources/chaos/chaos.go | 6 +++++- pkg/subscraping/sources/chinaz/chinaz.go | 6 +++++- .../sources/commoncrawl/commoncrawl.go | 6 +++++- pkg/subscraping/sources/crtsh/crtsh.go | 6 +++++- .../sources/digitalyama/digitalyama.go | 6 +++++- pkg/subscraping/sources/digitorus/digitorus.go | 6 +++++- pkg/subscraping/sources/dnsdb/dnsdb.go | 6 +++++- .../sources/dnsdumpster/dnsdumpster.go | 6 +++++- pkg/subscraping/sources/dnsrepo/dnsrepo.go | 6 +++++- .../sources/domainsproject/domainsproject.go | 6 +++++- pkg/subscraping/sources/driftnet/driftnet.go | 7 ++++++- pkg/subscraping/sources/facebook/ctlogs.go | 7 ++++++- pkg/subscraping/sources/fofa/fofa.go | 6 +++++- pkg/subscraping/sources/fullhunt/fullhunt.go | 6 +++++- pkg/subscraping/sources/github/github.go | 6 +++++- pkg/subscraping/sources/gitlab/gitlab.go | 6 +++++- .../sources/hackertarget/hackertarget.go | 14 +++++++------- .../sources/hudsonrock/hudsonrock.go | 6 +++++- pkg/subscraping/sources/intelx/intelx.go | 6 +++++- pkg/subscraping/sources/leakix/leakix.go | 6 +++++- pkg/subscraping/sources/merklemap/merklemap.go | 6 +++++- pkg/subscraping/sources/netlas/netlas.go | 6 +++++- pkg/subscraping/sources/onyphe/onyphe.go | 6 +++++- pkg/subscraping/sources/profundis/profundis.go | 6 +++++- pkg/subscraping/sources/pugrecon/pugrecon.go | 7 ++++++- pkg/subscraping/sources/quake/quake.go | 6 +++++- pkg/subscraping/sources/rapiddns/rapiddns.go | 6 +++++- .../sources/reconcloud/reconcloud.go | 6 +++++- pkg/subscraping/sources/reconeer/reconeer.go | 18 ++++++++++-------- .../sources/redhuntlabs/redhuntlabs.go | 6 +++++- pkg/subscraping/sources/riddler/riddler.go | 6 +++++- pkg/subscraping/sources/robtex/robtext.go | 6 +++++- pkg/subscraping/sources/rsecloud/rsecloud.go | 6 +++++- .../sources/securitytrails/securitytrails.go | 6 +++++- pkg/subscraping/sources/shodan/shodan.go | 6 +++++- .../sources/sitedossier/sitedossier.go | 6 +++++- pkg/subscraping/sources/thc/thc.go | 6 +++++- .../sources/threatbook/threatbook.go | 6 +++++- .../sources/threatcrowd/threatcrowd.go | 7 ++++++- .../sources/threatminer/threatminer.go | 6 +++++- .../sources/virustotal/virustotal.go | 6 +++++- .../sources/waybackarchive/waybackarchive.go | 6 +++++- .../sources/whoisxmlapi/whoisxmlapi.go | 6 +++++- pkg/subscraping/sources/windvane/windvane.go | 6 +++++- .../sources/zoomeyeapi/zoomeyeapi.go | 6 +++++- pkg/subscraping/types.go | 15 ++++++++++++++- 57 files changed, 305 insertions(+), 75 deletions(-) diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 292d60656..97877f85c 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -184,9 +184,9 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup } } - // TODO: Consider refactoring this to avoid potential duplication issues for _, source := range sources { - if source.NeedsKey() { + keyReq := source.KeyRequirement() + if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" { source.AddApiKeys([]string{apiKey}) } diff --git a/pkg/runner/config.go b/pkg/runner/config.go index fe6a3da40..b0c65205f 100644 --- a/pkg/runner/config.go +++ b/pkg/runner/config.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" fileutil "github.com/projectdiscovery/utils/file" ) @@ -25,7 +26,8 @@ func createProviderConfigYAML(configFilePath string) error { sourcesRequiringApiKeysMap := make(map[string][]string) for _, source := range passive.AllSources { - if source.NeedsKey() { + keyReq := source.KeyRequirement() + if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { sourceName := strings.ToLower(source.Name()) sourcesRequiringApiKeysMap[sourceName] = []string{} } diff --git a/pkg/runner/options.go b/pkg/runner/options.go index a2fd4d411..d3284cabf 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" envutil "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" @@ -223,16 +224,20 @@ func (options *Options) loadProvidersFrom(location string) { func listSources(options *Options) { gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) - gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n") + gologger.Info().Msgf("Sources marked with an * require key(s) or token(s) to work.\n") + gologger.Info().Msgf("Sources marked with a ~ optionally support key(s) for better results.\n") gologger.Info().Msgf("You can modify %s to configure your keys/tokens.\n\n", options.ProviderConfig) for _, source := range passive.AllSources { - message := "%s\n" sourceName := source.Name() - if source.NeedsKey() { - message = "%s *\n" + switch source.KeyRequirement() { + case subscraping.RequiredKey: + gologger.Silent().Msgf("%s *\n", sourceName) + case subscraping.OptionalKey: + gologger.Silent().Msgf("%s ~\n", sourceName) + default: + gologger.Silent().Msgf("%s\n", sourceName) } - gologger.Silent().Msgf(message, sourceName) } } diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go index 6af4268f2..f53f9959c 100644 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -98,8 +98,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go index 96d21e784..186055cc9 100644 --- a/pkg/subscraping/sources/anubis/anubis.go +++ b/pkg/subscraping/sources/anubis/anubis.go @@ -82,8 +82,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go index cbdcf0f43..98fb4188a 100644 --- a/pkg/subscraping/sources/bevigil/bevigil.go +++ b/pkg/subscraping/sources/bevigil/bevigil.go @@ -94,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 198d31e0b..d37cbd1f6 100644 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -119,8 +119,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go index a22043375..c49e349d0 100644 --- a/pkg/subscraping/sources/builtwith/builtwith.go +++ b/pkg/subscraping/sources/builtwith/builtwith.go @@ -100,8 +100,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go index 124686d27..8e165f5dc 100644 --- a/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -107,8 +107,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 388cc301c..84a8e1b21 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -197,8 +197,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys parses and adds API keys. diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index 35118ff65..5d1f2de78 100644 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -140,8 +140,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go index 489e3173a..1442d28ab 100644 --- a/pkg/subscraping/sources/chaos/chaos.go +++ b/pkg/subscraping/sources/chaos/chaos.go @@ -74,8 +74,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go index 2fd6c9546..5b5b55927 100644 --- a/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -91,8 +91,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index b1dda4910..8386b2828 100644 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -105,8 +105,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index caea79c42..4c3b53112 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -200,8 +200,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go index 8a89e686e..57b659ff8 100644 --- a/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -119,8 +119,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go index 91b3ca5b3..4cd10daa9 100644 --- a/pkg/subscraping/sources/digitorus/digitorus.go +++ b/pkg/subscraping/sources/digitorus/digitorus.go @@ -87,8 +87,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go index 810d8a05b..19ab65bb1 100644 --- a/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -183,8 +183,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 8062a4954..af25cb70d 100644 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -91,8 +91,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go index f7faaaac0..a0705c25a 100644 --- a/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -100,8 +100,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go index 800727d78..2d4d9bf3e 100644 --- a/pkg/subscraping/sources/domainsproject/domainsproject.go +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -117,8 +117,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index fa0d0bcdd..36ac8d8d0 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -104,9 +104,14 @@ func (s *Source) HasRecursiveSupport() bool { return true } +// KeyRequirement indicates that we need an API key +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + // NeedsKey indicates that we need an API key func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys provides us with the API key(s) diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index f0f37a4e0..6c123a548 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -164,9 +164,14 @@ func (s *Source) HasRecursiveSupport() bool { return true } +// KeyRequirement returns the API key requirement level for this source +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + // NeedsKey returns true if the source requires an API key func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys adds api keys to the source diff --git a/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go index b885476aa..8dd130f09 100644 --- a/pkg/subscraping/sources/fofa/fofa.go +++ b/pkg/subscraping/sources/fofa/fofa.go @@ -117,8 +117,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go index c53206e11..b89ec7864 100644 --- a/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -86,8 +86,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go index 69f5e151c..1a38dae56 100644 --- a/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -265,8 +265,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go index e2a13a111..66ca0f61f 100644 --- a/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -160,8 +160,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index f85b4fe88..4c25de9b0 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -33,13 +33,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return + if randomApiKey != "" { + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) } - htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) - resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -94,11 +91,14 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } -// TODO: env variable will not work if NeedsKey is false, entire api key management needs to be refactored func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } diff --git a/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go index b3e2e0ec8..ca77fa45b 100644 --- a/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -87,8 +87,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go index 82ecaeb16..a4b7c3d7b 100644 --- a/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -165,8 +165,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go index 0d3526ab7..f8525e0f3 100644 --- a/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -88,8 +88,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go index aa4881e72..23a455325 100644 --- a/pkg/subscraping/sources/merklemap/merklemap.go +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -139,8 +139,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go index a0c8b86e7..1886cd6a7 100644 --- a/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -184,8 +184,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/onyphe/onyphe.go b/pkg/subscraping/sources/onyphe/onyphe.go index a304ebdb3..23e61d365 100644 --- a/pkg/subscraping/sources/onyphe/onyphe.go +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -147,8 +147,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go index 698b98ac0..6a9f4cdd3 100644 --- a/pkg/subscraping/sources/profundis/profundis.go +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -109,8 +109,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go index 855154153..83d286023 100644 --- a/pkg/subscraping/sources/pugrecon/pugrecon.go +++ b/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -133,9 +133,14 @@ func (s *Source) HasRecursiveSupport() bool { return false } +// KeyRequirement returns the API key requirement level for this source. +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + // NeedsKey returns true as this source requires an API key. func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys adds the API keys for the source. diff --git a/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go index 2e99fb87f..dac5498e1 100644 --- a/pkg/subscraping/sources/quake/quake.go +++ b/pkg/subscraping/sources/quake/quake.go @@ -139,8 +139,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go index eb7969795..07234f97c 100644 --- a/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -102,8 +102,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go index 61bb593da..4778a962f 100644 --- a/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -89,8 +89,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/reconeer/reconeer.go b/pkg/subscraping/sources/reconeer/reconeer.go index 6463da9a5..934c681af 100644 --- a/pkg/subscraping/sources/reconeer/reconeer.go +++ b/pkg/subscraping/sources/reconeer/reconeer.go @@ -36,14 +36,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } headers := map[string]string{ - "Accept": "application/json", - "X-API-KEY": randomApiKey, + "Accept": "application/json", + } + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["X-API-KEY"] = randomApiKey } apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) resp, err := session.Get(ctx, apiURL, "", headers) @@ -92,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index 009d5d000..6ccd25d2b 100644 --- a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -136,8 +136,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go index 7855a344b..8475b0fac 100644 --- a/pkg/subscraping/sources/riddler/riddler.go +++ b/pkg/subscraping/sources/riddler/riddler.go @@ -78,8 +78,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/robtex/robtext.go b/pkg/subscraping/sources/robtex/robtext.go index f049d97ed..25f4e0418 100644 --- a/pkg/subscraping/sources/robtex/robtext.go +++ b/pkg/subscraping/sources/robtex/robtext.go @@ -131,8 +131,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go index 43d270b05..c9088555c 100644 --- a/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -108,8 +108,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index 0e6846e55..03337def9 100644 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -142,8 +142,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index b2195e393..6b02f2112 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -115,8 +115,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index dde639c40..abe2d6fe3 100644 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -97,8 +97,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go index 238062193..24390cc93 100644 --- a/pkg/subscraping/sources/thc/thc.go +++ b/pkg/subscraping/sources/thc/thc.go @@ -109,8 +109,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go index 6a6645d3b..4a8d65936 100644 --- a/pkg/subscraping/sources/threatbook/threatbook.go +++ b/pkg/subscraping/sources/threatbook/threatbook.go @@ -113,8 +113,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index 831659f61..9591c75d0 100644 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -108,9 +108,14 @@ func (s *Source) HasRecursiveSupport() bool { return false } +// KeyRequirement returns the API key requirement level for this source. +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + // NeedsKey indicates if the source requires an API key. func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys is a no-op since ThreatCrowd does not require an API key. diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 77f5a586e..11bd7c3b8 100644 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -80,8 +80,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 446c7c309..053f0ec70 100644 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -113,8 +113,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index ecbe4d943..e1000b99b 100644 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -84,8 +84,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { diff --git a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index c4efbdd2e..1949fa185 100644 --- a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -99,8 +99,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go index 92f972c40..49ab055cb 100644 --- a/pkg/subscraping/sources/windvane/windvane.go +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -136,8 +136,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 70d491c60..4982369cc 100644 --- a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -118,8 +118,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 454ad776f..edb600744 100644 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -33,6 +33,15 @@ type Statistics struct { Skipped bool } +// KeyRequirement represents the API key requirement level for a source +type KeyRequirement int + +const ( + NoKey KeyRequirement = iota + OptionalKey + RequiredKey +) + // Source is an interface inherited by each passive source type Source interface { // Run takes a domain as argument and a session object @@ -52,7 +61,11 @@ type Source interface { // not just root domains. HasRecursiveSupport() bool - // NeedsKey returns true if the source requires an API key + // KeyRequirement returns the API key requirement level for this source + KeyRequirement() KeyRequirement + + // NeedsKey returns true if the source requires an API key. + // Deprecated: Use KeyRequirement() instead for more granular control. NeedsKey() bool AddApiKeys([]string) From 1eac4a2b13705a1d142bf3c9ec48375c12f61685 Mon Sep 17 00:00:00 2001 From: spameier <40004508+spameier@users.noreply.github.com> Date: Wed, 21 Jan 2026 21:38:35 +0100 Subject: [PATCH 110/132] fix: mark LeakIX as requiring an API key --- pkg/subscraping/sources/leakix/leakix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go index f9a352aad..84c07b4d0 100644 --- a/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -92,7 +92,7 @@ func (s *Source) HasRecursiveSupport() bool { } func (s *Source) KeyRequirement() subscraping.KeyRequirement { - return subscraping.OptionalKey + return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { From d9f08bdc642c9595ea2d71ce6139ab48fad1e289 Mon Sep 17 00:00:00 2001 From: Mikel Olasagasti Uranga Date: Sat, 24 Jan 2026 10:36:27 +0100 Subject: [PATCH 111/132] dependanbot: change path after #1613 (#1708) --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba33d912a..63f4b632f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,7 +8,7 @@ updates: # Maintain dependencies for go modules - package-ecosystem: "gomod" - directory: "v2/" + directory: "/" schedule: interval: "weekly" target-branch: "dev" @@ -45,4 +45,4 @@ updates: # prefix: "chore" # include: "scope" # labels: -# - "Type: Maintenance" \ No newline at end of file +# - "Type: Maintenance" From 227ef90b2fa3b3c72c828a97f381dc9ca09475ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Jan 2026 15:11:49 +0530 Subject: [PATCH 112/132] chore(deps): bump the modules group with 10 updates (#1714) Bumps the modules group with 10 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/dnsx](https://github.com/projectdiscovery/dnsx) | `1.2.2` | `1.2.3` | | [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) | `1.1.54` | `1.1.62` | | [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.81` | `0.0.82` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.115` | `1.1.0` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.21` | `0.7.3` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.1.24` | `1.2.13` | | [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.4.1` | `0.4.19` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.90` | `0.0.98` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.16` | `0.1.31` | | [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) | `1.0.102` | `1.0.111` | Updates `github.com/projectdiscovery/dnsx` from 1.2.2 to 1.2.3 - [Release notes](https://github.com/projectdiscovery/dnsx/releases) - [Commits](https://github.com/projectdiscovery/dnsx/compare/v1.2.2...v1.2.3) Updates `github.com/projectdiscovery/gologger` from 1.1.54 to 1.1.62 - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.54...v1.1.62) Updates `github.com/projectdiscovery/ratelimit` from 0.0.81 to 0.0.82 - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.81...v0.0.82) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.115 to 1.1.0 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.115...v1.1.0) Updates `github.com/projectdiscovery/utils` from 0.4.21 to 0.7.3 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.21...v0.7.3) Updates `github.com/projectdiscovery/cdncheck` from 1.1.24 to 1.2.13 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.1.24...v1.2.13) Updates `github.com/projectdiscovery/fastdialer` from 0.4.1 to 0.4.19 - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.4.1...v0.4.19) Updates `github.com/projectdiscovery/hmap` from 0.0.90 to 0.0.98 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.90...v0.0.98) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.16 to 0.1.31 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.16...v0.1.31) Updates `github.com/projectdiscovery/retryabledns` from 1.0.102 to 1.0.111 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.102...v1.0.111) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dnsx dependency-version: 1.2.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/gologger dependency-version: 1.1.62 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/ratelimit dependency-version: 0.0.82 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.7.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.2.13 dependency-type: indirect update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/fastdialer dependency-version: 0.4.19 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.98 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.31 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.111 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 28 ++++++++++++------------ go.sum | 69 ++++++++++++++++++++++++---------------------------------- 2 files changed, 42 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 8ef3f30a1..5eb978410 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.2 - github.com/projectdiscovery/dnsx v1.2.2 + github.com/projectdiscovery/dnsx v1.2.3 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.54 - github.com/projectdiscovery/ratelimit v0.0.81 - github.com/projectdiscovery/retryablehttp-go v1.0.115 - github.com/projectdiscovery/utils v0.4.21 + github.com/projectdiscovery/gologger v1.1.62 + github.com/projectdiscovery/ratelimit v0.0.82 + github.com/projectdiscovery/retryablehttp-go v1.1.0 + github.com/projectdiscovery/utils v0.7.3 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 gopkg.in/yaml.v3 v3.0.1 @@ -49,7 +49,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaissmai/bart v0.20.4 // indirect + github.com/gaissmai/bart v0.26.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect @@ -77,12 +77,12 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.24 // indirect - github.com/projectdiscovery/fastdialer v0.4.1 // indirect - github.com/projectdiscovery/hmap v0.0.90 // indirect + github.com/projectdiscovery/cdncheck v1.2.13 // indirect + github.com/projectdiscovery/fastdialer v0.4.19 // indirect + github.com/projectdiscovery/hmap v0.0.98 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.16 // indirect - github.com/refraction-networking/utls v1.7.0 // indirect + github.com/projectdiscovery/networkpolicy v0.1.31 // indirect + github.com/refraction-networking/utls v1.7.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect @@ -101,7 +101,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/ulikunitz/xz v0.5.15 // indirect - github.com/weppos/publicsuffix-go v0.30.1 // indirect + github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -132,7 +132,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.102 // indirect + github.com/projectdiscovery/retryabledns v1.0.111 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index e56e41e07..376070824 100644 --- a/go.sum +++ b/go.sum @@ -26,7 +26,6 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= @@ -59,7 +58,6 @@ github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= @@ -75,7 +73,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= @@ -103,8 +100,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= -github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= +github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0= +github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -142,7 +139,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= -github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -255,37 +251,37 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.24 h1:6pJ4XnovIrTWzlCJs5/QD1tv6wvK0wiICmmdY0/8WAs= -github.com/projectdiscovery/cdncheck v1.1.24/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.2.13 h1:6zs4Mn8JV3yKyMoAr857Hf2NLvyOMpOfqCCT2V2OI1Q= +github.com/projectdiscovery/cdncheck v1.2.13/go.mod h1:/OhuZ9T25yXSqU6+oWvmVQ3QFvtew/Tp03u0jM+NJBE= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= -github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= -github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= -github.com/projectdiscovery/fastdialer v0.4.1 h1:kp6Q0odo0VZ0vZIGOn+q9aLgBSk6uYoD1MsjCAH8+h4= -github.com/projectdiscovery/fastdialer v0.4.1/go.mod h1:875Wlggf0JAz+fDIPwUQeeBqEF6nJA71XVrjuTZCV7I= +github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc= +github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE= +github.com/projectdiscovery/fastdialer v0.4.19 h1:MLHwEGM0x0pyltJaNvAVvwc27bnXdZ5mYr50S/2kMEE= +github.com/projectdiscovery/fastdialer v0.4.19/go.mod h1:HGdVsez+JgJ9/ljXjHRplOqkB7og+nqi0nrNWVNi03o= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= -github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.90 h1:p8HWGvPI88hgJoAb4ayR1Oo5VzqPrOCdFG7mASUhQI4= -github.com/projectdiscovery/hmap v0.0.90/go.mod h1:dcjd9P82mkBpFGEy0wBU/3qql5Bx14kmJZvVg7o7vXY= +github.com/projectdiscovery/gologger v1.1.62 h1:wzKqvL6HQRzf0/PpBEhInZqqL1q4mKe2gFGJeDG3FqE= +github.com/projectdiscovery/gologger v1.1.62/go.mod h1:YWvMSxlHybU3SkFCcWn+driSJ8yY+3CR3g/textnp+Y= +github.com/projectdiscovery/hmap v0.0.98 h1:XxYIi7yJCNiDAKCJXvuY9IBM5O6OgDgx4XHgKxkR4eg= +github.com/projectdiscovery/hmap v0.0.98/go.mod h1:bgN5fuZPJMj2YnAGEEnCypoifCnALJixHEVQszktQIU= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.16 h1:H2VnLmMD7SvxF+rao+639nn8KX/kbPFY+mc8FxeltsI= -github.com/projectdiscovery/networkpolicy v0.1.16/go.mod h1:Vs/IRcJq4QUicjd/tl9gkhQWy7d/LssOwWbaz4buJ0U= -github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= -github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= -github.com/projectdiscovery/retryabledns v1.0.102 h1:R8PzFCVofqLX3Bn4kdjOsE9wZ83FQjXZMDNs4/bHxzI= -github.com/projectdiscovery/retryabledns v1.0.102/go.mod h1:3+GL+YuHpV0Fp6UG7MbIG8mVxXHjfPO5ioQdwlnV08E= -github.com/projectdiscovery/retryablehttp-go v1.0.115 h1:ubIaVyHNj0/qxNv4gar+8/+L3G2Fhpfk54iMDctC7+E= -github.com/projectdiscovery/retryablehttp-go v1.0.115/go.mod h1:XlxLSMBVM7fTXeLVOLjVn1FLuRgQtD49NMFs9sQygfA= -github.com/projectdiscovery/utils v0.4.21 h1:yAothTUSF6NwZ9yoC4iGe5gSBrovqKR9JwwW3msxk3Q= -github.com/projectdiscovery/utils v0.4.21/go.mod h1:HJuJFqjB6EmVaDl0ilFPKvLoMaX2GyE6Il2TqKXNs8I= +github.com/projectdiscovery/networkpolicy v0.1.31 h1:mE6iJeYOSql8gps/91vwiztE/kEHe5Im8oUO5Mkj9Zg= +github.com/projectdiscovery/networkpolicy v0.1.31/go.mod h1:5x4rGh4XhnoYl9wACnZyrjDGKIB/bQqxw2KrIM5V+XU= +github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak= +github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= +github.com/projectdiscovery/retryabledns v1.0.111 h1:iyMdCDgNmaSRJYcGqB+SLlvlw9WijlbJ6Q9OEpRAWsQ= +github.com/projectdiscovery/retryabledns v1.0.111/go.mod h1:6TOPJ3QAE4reBu6bvsGsTcyEb+OypcKYFQH7yVsjyIM= +github.com/projectdiscovery/retryablehttp-go v1.1.0 h1:uYp3EnuhhamTwvG41X6q6TAc/SHEO9pw9CBWbRASIQk= +github.com/projectdiscovery/retryablehttp-go v1.1.0/go.mod h1:9DU57ezv5cfZSWw/m5XFDTMjy1yKeMyn1kj35lPlcfM= +github.com/projectdiscovery/utils v0.7.3 h1:kX+77AA58yK6EZgkTRJEnK9V/7AZYzlXdcu/o/kJhFs= +github.com/projectdiscovery/utils v0.7.3/go.mod h1:uDdQ3/VWomai98l+a3Ye/srDXdJ4xUIar/mSXlQ9gBM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= -github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= +github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= +github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -322,8 +318,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= @@ -363,8 +359,8 @@ github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= -github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= +github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 h1:Bz/zVM/LoGZ9IztGBHrq2zlFQQbEG8dBYnxb4hamIHM= +github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39/go.mod h1:2oFzEwGYI7lhiqG0YkkcKa6VcpjVinQbWxaPzytDmLA= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= @@ -414,7 +410,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -470,8 +465,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -480,7 +473,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -516,7 +508,6 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -534,8 +525,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -548,8 +537,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From b71a21bd9bb1ea15ec3f4c7954b53c5616377c9d Mon Sep 17 00:00:00 2001 From: PontusLindblom Date: Thu, 29 Jan 2026 18:04:58 +0100 Subject: [PATCH 113/132] Sort sources alphabetically Ensure the order of -ls matches provider-config.yaml to avoid confusion when comparing them. --- pkg/passive/sources.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 97877f85c..5567274b1 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -66,6 +66,7 @@ var AllSources = [...]subscraping.Source{ &anubis.Source{}, &bevigil.Source{}, &bufferover.Source{}, + &builtwith.Source{}, &c99.Source{}, &censys.Source{}, &certspotter.Source{}, @@ -73,47 +74,46 @@ var AllSources = [...]subscraping.Source{ &chinaz.Source{}, &commoncrawl.Source{}, &crtsh.Source{}, + &digitalyama.Source{}, &digitorus.Source{}, &dnsdb.Source{}, &dnsdumpster.Source{}, - &domainsproject.Source{}, &dnsrepo.Source{}, + &domainsproject.Source{}, &driftnet.Source{}, + &facebook.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, &hackertarget.Source{}, + &hudsonrock.Source{}, &intelx.Source{}, - &netlas.Source{}, + &leakix.Source{}, &merklemap.Source{}, + &netlas.Source{}, &onyphe.Source{}, - &leakix.Source{}, - &quake.Source{}, + &profundis.Source{}, &pugrecon.Source{}, + &quake.Source{}, &rapiddns.Source{}, + // &reconcloud.Source{}, // failing due to cloudflare bot protection + &reconeer.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, &rsecloud.Source{}, &securitytrails.Source{}, - &profundis.Source{}, &shodan.Source{}, &sitedossier.Source{}, + &thc.Source{}, &threatbook.Source{}, &threatcrowd.Source{}, + // &threatminer.Source{}, // failing api &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, &windvane.Source{}, &zoomeyeapi.Source{}, - &facebook.Source{}, - // &threatminer.Source{}, // failing api - // &reconcloud.Source{}, // failing due to cloudflare bot protection - &reconeer.Source{}, - &builtwith.Source{}, - &hudsonrock.Source{}, - &digitalyama.Source{}, - &thc.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( From 978addb8bd190e3c9d5a526e4bf619fd1b6c9eb1 Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 17:02:44 +0530 Subject: [PATCH 114/132] feat: add URLScan.io as passive subdomain source Add URLScan.io as a new passive subdomain enumeration source with full pagination support and robust rate limiting handling. Features: - Fetches subdomains from URLScan.io Search API - Implements cursor-based pagination using search_after parameter - Extracts domains from task.domain, task.url, page.domain, page.url fields - Requires API key (free tier available at urlscan.io) Rate Limiting: - Conservative pagination delay (10s between pages) to respect strict burst limits - Exponential backoff retry logic for 429/503 responses - Respects X-Rate-Limit-Reset-After header for dynamic backoff - Limited to 5 pages max (500 results) to avoid quota exhaustion Configuration: - Max 5 pages per enumeration (configurable via maxPages constant) - 100 results per page (configurable via maxPerPage constant) - 2 retry attempts for rate-limited requests - 20 second initial backoff, doubles on each retry Changes: - pkg/subscraping/sources/urlscan/urlscan.go: New URLScan source implementation - pkg/passive/sources.go: Register URLScan source - pkg/passive/sources_test.go: Add URLScan to test lists - pkg/runner/options.go: Add urlscan to source options - .github/workflows/build-test.yml: Add URLSCAN_API_KEY secret Closes: Feature request for URLScan.io integration --- .github/workflows/build-test.yml | 1 + pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 3 + pkg/runner/options.go | 1 + pkg/subscraping/sources/urlscan/urlscan.go | 295 +++++++++++++++++++++ 5 files changed, 302 insertions(+) create mode 100644 pkg/subscraping/sources/urlscan/urlscan.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d567636b7..c6728ea51 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -65,6 +65,7 @@ jobs: SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} + URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 97877f85c..c5d3a5148 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -53,6 +53,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" @@ -114,6 +115,7 @@ var AllSources = [...]subscraping.Source{ &hudsonrock.Source{}, &digitalyama.Source{}, &thc.Source{}, + &urlscan.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 82aa0336b..7c4c0da30 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -63,6 +63,7 @@ var ( "digitalyama", "merklemap", "thc", + "urlscan", } expectedDefaultSources = []string{ @@ -105,6 +106,7 @@ var ( "builtwith", "digitalyama", "thc", + "urlscan", } expectedDefaultRecursiveSources = []string{ @@ -121,6 +123,7 @@ var ( "leakix", "facebook", "merklemap", + "urlscan", // "reconcloud", } ) diff --git a/pkg/runner/options.go b/pkg/runner/options.go index d3284cabf..946c3027b 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -265,4 +265,5 @@ var defaultRateLimits = []string{ // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", + "urlscan=1/s", } diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go new file mode 100644 index 000000000..584acbbd7 --- /dev/null +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -0,0 +1,295 @@ +// Package urlscan logic +package urlscan + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the URLScan API base URL + baseURL = "https://urlscan.io/api/v1/search/" + // maxPages is the maximum number of pages to fetch (reduced due to strict rate limits) + // URLScan has very strict burst limiting, so we limit to 3-5 pages max + maxPages = 5 + // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) + maxPerPage = 100 + // maxRetries is the number of retry attempts for rate-limited requests + // Reducing retries since each retry consumes quota + maxRetries = 2 + // initialBackoff is the initial wait time before retrying (URLScan recommends waiting) + initialBackoff = 20 * time.Second + // paginationDelay is the delay between pagination requests to avoid rate limits + // URLScan has 120 requests/minute but very strict burst limits + // Using 8-10 seconds to be extra conservative + paginationDelay = 10 * time.Second +) + +// response represents the URLScan API response structure +type response struct { + Results []struct { + Task struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"task"` + Page struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"page"` + Sort []interface{} `json:"sort"` + } `json:"results"` + HasMore bool `json:"has_more"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + s.skipped = false + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"api-key": randomApiKey} + + // Search with wildcard to get more subdomain results + s.enumerate(ctx, domain, headers, session, results) + }() + + return results +} + +// enumerate performs the actual enumeration with pagination +func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + var searchAfter string + currentPage := 0 + + for { + // Check context at the start of each iteration (standard pattern) + select { + case <-ctx.Done(): + return + default: + } + + // Check max pages limit (similar to Censys) + if currentPage >= maxPages { + break + } + + // Build search URL - search for domain and all subdomains + searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) + if searchAfter != "" { + searchURL += "&search_after=" + url.QueryEscape(searchAfter) + } + + resp, err := s.makeRequestWithRetry(ctx, session, searchURL, headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + session.DiscardHTTPResponse(resp) + + // Process results - extract subdomains from multiple fields + for _, result := range data.Results { + // Extract from task.domain, task.url, page.domain, page.url + candidates := []string{ + result.Task.Domain, + result.Page.Domain, + } + + // Also extract from URLs if present + if result.Task.URL != "" { + if u, err := url.Parse(result.Task.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + if result.Page.URL != "" { + if u, err := url.Parse(result.Page.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + + for _, candidate := range candidates { + if candidate == "" { + continue + } + for _, subdomain := range session.Extractor.Extract(candidate) { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + } + + // Check pagination conditions (similar to Shodan/VirusTotal pattern) + if !data.HasMore || len(data.Results) == 0 { + break + } + + // Get sort value for next page + lastResult := data.Results[len(data.Results)-1] + if len(lastResult.Sort) == 0 { + break + } + + // Build search_after parameter + sortValues := make([]string, len(lastResult.Sort)) + for i, v := range lastResult.Sort { + switch val := v.(type) { + case float64: + sortValues[i] = fmt.Sprintf("%.0f", val) + default: + sortValues[i] = fmt.Sprintf("%v", v) + } + } + // Don't URL encode here - the session.Get will handle encoding + searchAfter = strings.Join(sortValues, ",") + currentPage++ + + // Delay between pages to respect rate limits + select { + case <-ctx.Done(): + return + case <-time.After(paginationDelay): + } + } +} + +// makeRequestWithRetry handles rate limiting (429) and server errors (503) with exponential backoff +// This is necessary for URLScan as it has stricter rate limits than other sources +func (s *Source) makeRequestWithRetry(ctx context.Context, session *subscraping.Session, searchURL string, headers map[string]string) (*http.Response, error) { + var resp *http.Response + var err error + backoff := initialBackoff + + for attempt := 0; attempt <= maxRetries; attempt++ { + s.requests++ + resp, err = session.Get(ctx, searchURL, "", headers) + + // If request succeeded, return it + if err == nil && resp != nil && resp.StatusCode == http.StatusOK { + return resp, nil + } + + // Check for retryable status codes (429 rate limited, 503 server overload) + if resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable) { + // Try to get recommended wait time from X-Rate-Limit-Reset-After header + resetAfter := resp.Header.Get("X-Rate-Limit-Reset-After") + if resetAfter != "" { + // Parse the seconds and use it as wait time + if seconds, parseErr := time.ParseDuration(resetAfter + "s"); parseErr == nil && seconds > 0 { + backoff = seconds + (2 * time.Second) // Add 2 extra seconds as buffer + } + } + + session.DiscardHTTPResponse(resp) + + if attempt < maxRetries { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff): + backoff *= 2 // Exponential backoff for next retry + } + continue + } + return nil, fmt.Errorf("rate limited (status %d) after %d retries", resp.StatusCode, maxRetries+1) + } + + // For other errors (non-200, non-retryable status codes), return an error + // This prevents callers from trying to decode error response bodies + if resp != nil { + status := resp.StatusCode + session.DiscardHTTPResponse(resp) + if err == nil { + err = fmt.Errorf("unexpected status %d", status) + } + } else if err == nil { + err = fmt.Errorf("unexpected nil response") + } + return nil, err + } + + // This return is required by the Go compiler but is technically unreachable + // since all paths in the loop above return + return nil, fmt.Errorf("max retries exceeded") +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "urlscan" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From e8c0e16d6db51a6f2fe7fa82b0dd23864cbc6f9b Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 17:42:20 +0530 Subject: [PATCH 115/132] fix: replace deprecated errorutil with fmt.Errorf in facebook source Replace deprecated github.com/projectdiscovery/utils/errors package with standard Go error wrapping using fmt.Errorf to fix staticcheck SA1019 linter errors. --- pkg/subscraping/sources/facebook/ctlogs.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index 444db88ea..dab71b8d7 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -10,7 +10,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - errorutil "github.com/projectdiscovery/utils/errors" "github.com/projectdiscovery/utils/generic" urlutil "github.com/projectdiscovery/utils/url" ) @@ -128,7 +127,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se response := &response{} if err := json.Unmarshal(bin, response); err != nil { s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to unmarshal response: %s: %w", string(bin), err)} return } for _, v := range response.Data { From 675be65d479a736cf2fe70b61f9d19b74f774a6e Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 17:57:10 +0530 Subject: [PATCH 116/132] test: skip flaky sources in TestSourcesWithoutKeys Add leakix, reconeer, and sitedossier to ignored sources list: - leakix: now requires API key (returns 401) - reconeer: now requires API key (returns 401) - sitedossier: flaky, returns no results in CI --- pkg/passive/sources_wo_auth_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go index 01fc57d98..c3316d63c 100644 --- a/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -34,6 +34,9 @@ func TestSourcesWithoutKeys(t *testing.T) { "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" + "leakix", // now requires API key (returns 401) + "reconeer", // now requires API key (returns 401) + "sitedossier", // flaky - returns no results in CI } domain := "hackerone.com" From 58c1e5ac829cc0aa4cc504ad795c2f4842f0348f Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 18:05:47 +0530 Subject: [PATCH 117/132] refactor: simplify urlscan source to use session rate limiting Remove custom pagination delay and retry logic since the session already handles rate limiting via MultiRateLimiter. This aligns with how other sources (shodan, virustotal) are implemented. --- pkg/subscraping/sources/urlscan/urlscan.go | 92 ++-------------------- 1 file changed, 6 insertions(+), 86 deletions(-) diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go index 584acbbd7..1b0a9fcff 100644 --- a/pkg/subscraping/sources/urlscan/urlscan.go +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -4,7 +4,6 @@ package urlscan import ( "context" "fmt" - "net/http" "net/url" "strings" "time" @@ -17,20 +16,10 @@ import ( const ( // baseURL is the URLScan API base URL baseURL = "https://urlscan.io/api/v1/search/" - // maxPages is the maximum number of pages to fetch (reduced due to strict rate limits) - // URLScan has very strict burst limiting, so we limit to 3-5 pages max + // maxPages is the maximum number of pages to fetch maxPages = 5 // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) maxPerPage = 100 - // maxRetries is the number of retry attempts for rate-limited requests - // Reducing retries since each retry consumes quota - maxRetries = 2 - // initialBackoff is the initial wait time before retrying (URLScan recommends waiting) - initialBackoff = 20 * time.Second - // paginationDelay is the delay between pagination requests to avoid rate limits - // URLScan has 120 requests/minute but very strict burst limits - // Using 8-10 seconds to be extra conservative - paginationDelay = 10 * time.Second ) // response represents the URLScan API response structure @@ -95,28 +84,28 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin currentPage := 0 for { - // Check context at the start of each iteration (standard pattern) select { case <-ctx.Done(): return default: } - // Check max pages limit (similar to Censys) if currentPage >= maxPages { break } - // Build search URL - search for domain and all subdomains + // Build search URL searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) if searchAfter != "" { searchURL += "&search_after=" + url.QueryEscape(searchAfter) } - resp, err := s.makeRequestWithRetry(ctx, session, searchURL, headers) + s.requests++ + resp, err := session.Get(ctx, searchURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ + session.DiscardHTTPResponse(resp) return } @@ -132,7 +121,6 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin // Process results - extract subdomains from multiple fields for _, result := range data.Results { - // Extract from task.domain, task.url, page.domain, page.url candidates := []string{ result.Task.Domain, result.Page.Domain, @@ -165,7 +153,7 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin } } - // Check pagination conditions (similar to Shodan/VirusTotal pattern) + // Check pagination conditions if !data.HasMore || len(data.Results) == 0 { break } @@ -186,77 +174,9 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin sortValues[i] = fmt.Sprintf("%v", v) } } - // Don't URL encode here - the session.Get will handle encoding searchAfter = strings.Join(sortValues, ",") currentPage++ - - // Delay between pages to respect rate limits - select { - case <-ctx.Done(): - return - case <-time.After(paginationDelay): - } - } -} - -// makeRequestWithRetry handles rate limiting (429) and server errors (503) with exponential backoff -// This is necessary for URLScan as it has stricter rate limits than other sources -func (s *Source) makeRequestWithRetry(ctx context.Context, session *subscraping.Session, searchURL string, headers map[string]string) (*http.Response, error) { - var resp *http.Response - var err error - backoff := initialBackoff - - for attempt := 0; attempt <= maxRetries; attempt++ { - s.requests++ - resp, err = session.Get(ctx, searchURL, "", headers) - - // If request succeeded, return it - if err == nil && resp != nil && resp.StatusCode == http.StatusOK { - return resp, nil - } - - // Check for retryable status codes (429 rate limited, 503 server overload) - if resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable) { - // Try to get recommended wait time from X-Rate-Limit-Reset-After header - resetAfter := resp.Header.Get("X-Rate-Limit-Reset-After") - if resetAfter != "" { - // Parse the seconds and use it as wait time - if seconds, parseErr := time.ParseDuration(resetAfter + "s"); parseErr == nil && seconds > 0 { - backoff = seconds + (2 * time.Second) // Add 2 extra seconds as buffer - } - } - - session.DiscardHTTPResponse(resp) - - if attempt < maxRetries { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(backoff): - backoff *= 2 // Exponential backoff for next retry - } - continue - } - return nil, fmt.Errorf("rate limited (status %d) after %d retries", resp.StatusCode, maxRetries+1) - } - - // For other errors (non-200, non-retryable status codes), return an error - // This prevents callers from trying to decode error response bodies - if resp != nil { - status := resp.StatusCode - session.DiscardHTTPResponse(resp) - if err == nil { - err = fmt.Errorf("unexpected status %d", status) - } - } else if err == nil { - err = fmt.Errorf("unexpected nil response") - } - return nil, err } - - // This return is required by the Go compiler but is technically unreachable - // since all paths in the loop above return - return nil, fmt.Errorf("max retries exceeded") } // Name returns the name of the source From fba1cc3d27fbcc24e2265b2b5a62967ff5a5ea95 Mon Sep 17 00:00:00 2001 From: Recep Gunes Date: Mon, 2 Feb 2026 18:13:11 +0300 Subject: [PATCH 118/132] fix timeout flag not working for crtsh source --- pkg/subscraping/agent.go | 2 +- pkg/subscraping/sources/crtsh/crtsh.go | 5 ++++- pkg/subscraping/types.go | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/subscraping/agent.go b/pkg/subscraping/agent.go index 4f2cab8c6..6217ce58d 100644 --- a/pkg/subscraping/agent.go +++ b/pkg/subscraping/agent.go @@ -46,7 +46,7 @@ func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLi Timeout: time.Duration(timeout) * time.Second, } - session := &Session{Client: client} + session := &Session{Client: client, Timeout: timeout} // Initiate rate limit instance session.MultiRateLimiter = multiRateLimiter diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index 2e235d14f..a22b5bdc7 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -56,7 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int { - db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") + // connect_timeout: limits connection establishment time (in seconds) + // statement_timeout: limits query execution time (in milliseconds) + connStr := fmt.Sprintf("host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes connect_timeout=%d statement_timeout=%d", session.Timeout, session.Timeout*1000) + db, err := sql.Open("postgres", connStr) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 92b3b4bb6..30d0f4073 100644 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -89,6 +89,8 @@ type Session struct { Client *http.Client // Rate limit instance MultiRateLimiter *ratelimit.MultiLimiter + // Timeout is the timeout in seconds for requests + Timeout int } // Result is a result structure returned by a source From 0b517626089bf884805b5056dfaf50b1fe89bb91 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:55:54 +0300 Subject: [PATCH 119/132] Add PR template (#1729) --- .github/PULL_REQUEST_TEMPLATE.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..ff63af57f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Proposed changes + + + +### Proof + + + +## Checklist + + + +- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/subfinder/tree/dev) branch +- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added necessary documentation (if appropriate) \ No newline at end of file From 0dfd80add54f2e8a767ca34bbe3213662c98e051 Mon Sep 17 00:00:00 2001 From: PontusLindblom Date: Sat, 7 Feb 2026 07:44:29 +0100 Subject: [PATCH 120/132] Update workflow dependencies --- .github/workflows/build-test.yml | 8 ++++---- .github/workflows/codeql-analysis.yml | 8 ++++---- .github/workflows/compat-checks.yaml | 5 ++--- .github/workflows/dep-auto-merge.yml | 4 ++-- .github/workflows/dockerhub-push.yml | 10 +++++----- .github/workflows/release-binary.yml | 12 ++++++------ .github/workflows/release-test.yml | 6 +++--- 7 files changed, 26 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c6728ea51..5956c1515 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -19,12 +19,12 @@ jobs: if: "${{ !endsWith(github.actor, '[bot]') }}" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 with: go-version-file: go.mod - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@v9 with: version: latest args: --timeout 5m @@ -37,7 +37,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 with: go-version-file: go.mod @@ -69,7 +69,7 @@ jobs: VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v3 with: timeout_seconds: 360 max_attempts: 3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 927b1cc11..31794f2a7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,16 +24,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/compat-checks.yaml b/.github/workflows/compat-checks.yaml index 8efb822d2..a19241221 100644 --- a/.github/workflows/compat-checks.yaml +++ b/.github/workflows/compat-checks.yaml @@ -13,8 +13,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go/compat-checks@master with: - go-version-file: 'go.mod' - + go-version-file: go.mod diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml index 84b26e1fe..a21b93389 100644 --- a/.github/workflows/dep-auto-merge.yml +++ b/.github/workflows/dep-auto-merge.yml @@ -16,11 +16,11 @@ jobs: runs-on: ubuntu-latest if: github.actor == 'dependabot[bot]' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: token: ${{ secrets.DEPENDABOT_PAT }} - uses: ahmadnassri/action-dependabot-auto-merge@v2 with: github-token: ${{ secrets.DEPENDABOT_PAT }} - target: all \ No newline at end of file + target: all diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 4409dcc30..582c32535 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Get Github tag id: meta @@ -20,19 +20,19 @@ jobs: curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64,linux/arm diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index ea68f5d0b..b6625c01c 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -11,15 +11,15 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: fetch-depth: 0 - + - name: "Set up Go" - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v3 with: diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 1c8fbbf8b..f5b8b74c0 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -12,14 +12,14 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v6 with: - go-version: 1.21.x + go-version-file: go.mod - name: release test uses: goreleaser/goreleaser-action@v4 From 13e5db750080522176fb3e4abaebd9806ef3680c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 5 Mar 2026 14:48:23 +0300 Subject: [PATCH 121/132] bump version --- pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index 4d7a9532c..a96fa70db 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.12.0` +const version = `v2.13.0` // showBanner is used to show the banner to the user func showBanner() { From c1ce78eac9ff2fbe1edda4b36f41a19781cdce68 Mon Sep 17 00:00:00 2001 From: Charles Wong Date: Tue, 10 Mar 2026 09:47:09 -0700 Subject: [PATCH 122/132] fix(ratelimit): honour -rl global fallback and -rls duration (#1434) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs in rate limit handling: 1. Global -rl flag was ignored for sources without a per-source override. Only sources explicitly listed in -rls got rate-limited; all others ran unlimited, defeating the purpose of -rl. 2. Per-source duration from -rls (e.g. hackertarget=2/m) was silently dropped. runner.go stored MaxCount but discarded Duration, so all sources defaulted to per-second regardless of the /m or /s suffix. Root cause: runner.go converted goflags.RateLimitMap β†’ CustomRateLimit but only copied MaxCount. buildMultiRateLimiter only checked per-source limits, never falling back to the global rate. Changes: - Add CustomDuration field to CustomRateLimit struct - Store both MaxCount and Duration from -rls in runner.go - Add resolveSourceRateLimit() with clear priority: per-source > global > unlimited - Guard against nil rateLimit in buildMultiRateLimiter - Add 9 unit tests covering all combinations Fixes #1434 --- pkg/passive/passive.go | 34 +++++++-- pkg/passive/ratelimit_test.go | 125 ++++++++++++++++++++++++++++++++++ pkg/runner/runner.go | 7 ++ pkg/subscraping/types.go | 3 +- 4 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 pkg/passive/ratelimit_test.go diff --git a/pkg/passive/passive.go b/pkg/passive/passive.go index 81a7cf041..9b2661ee9 100644 --- a/pkg/passive/passive.go +++ b/pkg/passive/passive.go @@ -85,14 +85,14 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error + if rateLimit == nil { + rateLimit = &subscraping.CustomRateLimit{} + } for _, source := range a.sources { - var rl uint - if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { - rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit) - } + rl, duration := resolveSourceRateLimit(globalRateLimit, rateLimit, source.Name()) if rl > 0 { - multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second) + multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, duration) } else { multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond) } @@ -104,6 +104,30 @@ func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, return multiRateLimiter, err } +// resolveSourceRateLimit returns the effective rate limit and duration for a source. +// Priority: per-source custom limit > global -rl limit > unlimited. +// Duration comes from -rls (e.g. hackertarget=2/m β†’ 2 per minute), defaulting to per-second. +func resolveSourceRateLimit(globalRateLimit int, rateLimit *subscraping.CustomRateLimit, sourceName string) (uint, time.Duration) { + duration := time.Second // default: requests per second + + sourceLower := strings.ToLower(sourceName) + if sourceRL, ok := rateLimit.Custom.Get(sourceLower); ok { + rl := sourceRateLimitOrDefault(uint(max(globalRateLimit, 0)), sourceRL) + // Use per-source duration from -rls if set (e.g. "2/m" β†’ time.Minute) + if d, ok := rateLimit.CustomDuration.Get(sourceLower); ok && d > 0 { + duration = d + } + return rl, duration + } + + // No per-source limit: fall back to global -rl + if globalRateLimit > 0 { + return uint(globalRateLimit), duration + } + + return 0, duration +} + func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint { if sourceRateLimit > 0 { return sourceRateLimit diff --git a/pkg/passive/ratelimit_test.go b/pkg/passive/ratelimit_test.go new file mode 100644 index 000000000..9cce7b572 --- /dev/null +++ b/pkg/passive/ratelimit_test.go @@ -0,0 +1,125 @@ +package passive + +import ( + "context" + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mapsutil "github.com/projectdiscovery/utils/maps" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +func newCustomRateLimit(counts map[string]uint, durations map[string]time.Duration) *subscraping.CustomRateLimit { + crl := &subscraping.CustomRateLimit{ + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, + } + for k, v := range counts { + _ = crl.Custom.Set(k, v) + } + for k, d := range durations { + _ = crl.CustomDuration.Set(k, d) + } + return crl +} + +func TestResolveSourceRateLimit_PerSourceOverride(t *testing.T) { + // Per-source limit should override global + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 5}, + map[string]time.Duration{"hackertarget": time.Minute}, + ) + + rl, dur := resolveSourceRateLimit(10, crl, "hackertarget") + assert.Equal(t, uint(5), rl) + assert.Equal(t, time.Minute, dur) +} + +func TestResolveSourceRateLimit_GlobalFallback(t *testing.T) { + // Sources without per-source limit should use global -rl + crl := newCustomRateLimit(nil, nil) + + rl, dur := resolveSourceRateLimit(10, crl, "crtsh") + assert.Equal(t, uint(10), rl) + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_NoLimit(t *testing.T) { + // No global and no per-source β†’ unlimited (0) + crl := newCustomRateLimit(nil, nil) + + rl, _ := resolveSourceRateLimit(0, crl, "crtsh") + assert.Equal(t, uint(0), rl) +} + +func TestResolveSourceRateLimit_NilRateLimit(t *testing.T) { + // nil CustomRateLimit should not panic + rl, dur := resolveSourceRateLimit(5, &subscraping.CustomRateLimit{}, "crtsh") + assert.Equal(t, uint(5), rl) + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_PerSourceDefaultDuration(t *testing.T) { + // Per-source limit without explicit duration defaults to per-second + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 3}, + nil, + ) + + rl, dur := resolveSourceRateLimit(0, crl, "hackertarget") + assert.Equal(t, uint(3), rl) + assert.Equal(t, time.Second, dur) +} + +func TestBuildMultiRateLimiter_GlobalAppliedToAll(t *testing.T) { + // With -rl 5 and no per-source overrides, all sources should be rate-limited + agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false) + crl := newCustomRateLimit(nil, nil) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 5, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_NilRateLimit(t *testing.T) { + // nil rateLimit should not panic (guard in buildMultiRateLimiter) + agent := New([]string{"hackertarget"}, []string{}, false, false) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, nil) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_PerSourceWithDuration(t *testing.T) { + // Per-source limit with custom duration (e.g. -rls hackertarget=2/m) + agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false) + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 2}, + map[string]time.Duration{"hackertarget": time.Minute}, + ) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_UnlimitedWhenNoLimits(t *testing.T) { + // Without any rate limits, sources should get MaxUint32 (effectively unlimited) + agent := New([]string{"hackertarget"}, []string{}, false, false) + crl := newCustomRateLimit(nil, nil) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) + require.NoError(t, err) + require.NotNil(t, limiter) + // No assertion on internal state - just verify it doesn't error + _ = math.MaxUint32 // referenced for clarity +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index f00ce6418..b9dd844d3 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -10,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/projectdiscovery/gologger" contextutil "github.com/projectdiscovery/utils/context" @@ -61,11 +62,17 @@ func NewRunner(options *Options) (*Runner, error) { Custom: mapsutil.SyncLockMap[string, uint]{ Map: make(map[string]uint), }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, } for source, sourceRateLimit := range options.RateLimits.AsMap() { if sourceRateLimit.MaxCount > 0 && sourceRateLimit.MaxCount <= math.MaxUint { _ = runner.rateLimit.Custom.Set(source, sourceRateLimit.MaxCount) + if sourceRateLimit.Duration > 0 { + _ = runner.rateLimit.CustomDuration.Set(source, sourceRateLimit.Duration) + } } } diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 30d0f4073..381187171 100644 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -16,7 +16,8 @@ const ( ) type CustomRateLimit struct { - Custom mapsutil.SyncLockMap[string, uint] + Custom mapsutil.SyncLockMap[string, uint] + CustomDuration mapsutil.SyncLockMap[string, time.Duration] } // BasicAuth request's Authorization header From 6f0e11ef05b01269924d379f770f9e9b5558b130 Mon Sep 17 00:00:00 2001 From: Charles Wong Date: Tue, 10 Mar 2026 14:09:07 -0700 Subject: [PATCH 123/132] refactor: rename misleading test per review feedback --- pkg/passive/ratelimit_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/passive/ratelimit_test.go b/pkg/passive/ratelimit_test.go index 9cce7b572..69da66cd0 100644 --- a/pkg/passive/ratelimit_test.go +++ b/pkg/passive/ratelimit_test.go @@ -61,8 +61,8 @@ func TestResolveSourceRateLimit_NoLimit(t *testing.T) { assert.Equal(t, uint(0), rl) } -func TestResolveSourceRateLimit_NilRateLimit(t *testing.T) { - // nil CustomRateLimit should not panic +func TestResolveSourceRateLimit_ZeroValueRateLimit(t *testing.T) { + // Zero-value CustomRateLimit (nil maps) should not panic and fall back to global rl, dur := resolveSourceRateLimit(5, &subscraping.CustomRateLimit{}, "crtsh") assert.Equal(t, uint(5), rl) assert.Equal(t, time.Second, dur) From cca10d8c9c105c4fabf13fdee2f69e0ee3b2164f Mon Sep 17 00:00:00 2001 From: Charles Wong Date: Tue, 10 Mar 2026 22:24:19 -0700 Subject: [PATCH 124/132] fix: initialize CustomRateLimit maps to prevent nil map panic CodeRabbit identified that the zero-valued CustomRateLimit struct left Custom.Map and CustomDuration.Map as nil, which could cause a panic when resolveSourceRateLimit calls .Get() on these maps. Mirror the initialization pattern from runner.go by explicitly creating the underlying maps. --- pkg/passive/passive.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/passive/passive.go b/pkg/passive/passive.go index 9b2661ee9..c00a024d5 100644 --- a/pkg/passive/passive.go +++ b/pkg/passive/passive.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + mapsutil "github.com/projectdiscovery/utils/maps" ) type EnumerationOptions struct { @@ -86,7 +87,14 @@ func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, var multiRateLimiter *ratelimit.MultiLimiter var err error if rateLimit == nil { - rateLimit = &subscraping.CustomRateLimit{} + rateLimit = &subscraping.CustomRateLimit{ + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, + } } for _, source := range a.sources { rl, duration := resolveSourceRateLimit(globalRateLimit, rateLimit, source.Name()) From cf0b8017c4edd8fdea37f982aa63502f113b7e15 Mon Sep 17 00:00:00 2001 From: Charles Wong Date: Wed, 11 Mar 2026 09:12:06 -0700 Subject: [PATCH 125/132] test: remove dead math.MaxUint32 no-op and unused math import Per CodeRabbit nitpick: the blank identifier assignment and math import in TestBuildMultiRateLimiter_UnlimitedWhenNoLimits served no purpose. The test already verifies correct behavior via NoError + NotNil assertions. --- pkg/passive/ratelimit_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/passive/ratelimit_test.go b/pkg/passive/ratelimit_test.go index 69da66cd0..0c0e95ae6 100644 --- a/pkg/passive/ratelimit_test.go +++ b/pkg/passive/ratelimit_test.go @@ -2,7 +2,6 @@ package passive import ( "context" - "math" "testing" "time" @@ -120,6 +119,4 @@ func TestBuildMultiRateLimiter_UnlimitedWhenNoLimits(t *testing.T) { limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) require.NoError(t, err) require.NotNil(t, limiter) - // No assertion on internal state - just verify it doesn't error - _ = math.MaxUint32 // referenced for clarity } From 240305b985aaeab2a557001cf2a342dc446f6796 Mon Sep 17 00:00:00 2001 From: Charles Wong Date: Wed, 11 Mar 2026 09:21:31 -0700 Subject: [PATCH 126/132] test: add edge case coverage for resolveSourceRateLimit - PerSourceExceedsGlobal: per-source limit > global should be honoured (verifies sourceRateLimitOrDefault returns source, not global) - CaseInsensitive: source names stored lowercase must match mixed-case lookups (documents and verifies the ToLower normalization contract) --- pkg/passive/ratelimit_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/passive/ratelimit_test.go b/pkg/passive/ratelimit_test.go index 0c0e95ae6..36c929e1e 100644 --- a/pkg/passive/ratelimit_test.go +++ b/pkg/passive/ratelimit_test.go @@ -120,3 +120,28 @@ func TestBuildMultiRateLimiter_UnlimitedWhenNoLimits(t *testing.T) { require.NoError(t, err) require.NotNil(t, limiter) } + +func TestResolveSourceRateLimit_PerSourceExceedsGlobal(t *testing.T) { + // Per-source limit higher than global should be honoured + // (the source was explicitly allowed more than the default) + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 20}, + nil, + ) + + rl, dur := resolveSourceRateLimit(10, crl, "hackertarget") + assert.Equal(t, uint(20), rl, "per-source limit should take precedence even when > global") + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_CaseInsensitive(t *testing.T) { + // Source names should be case-insensitive for lookup + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 5}, + nil, + ) + + // Try mixed case β€” should still match the lower-cased key + rl, _ := resolveSourceRateLimit(0, crl, "HackerTarget") + assert.Equal(t, uint(5), rl, "source name lookup should be case-insensitive") +} From 7bd72ff7f77bce6b5b49c2d3fc9dded2ebee7caf Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 20 Mar 2026 23:09:48 +0100 Subject: [PATCH 127/132] use org actions --- .github/workflows/build-test.yml | 10 +--------- .github/workflows/release-binary.yml | 10 +++------- .github/workflows/release-test.yml | 12 +++--------- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 5956c1515..939fc835d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -21,13 +21,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - with: - go-version-file: go.mod - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v9 - with: - version: latest - args: --timeout 5m + - uses: projectdiscovery/actions/golangci-lint/v2@v1 build: name: Test Builds @@ -39,8 +33,6 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - with: - go-version-file: go.mod - run: go build ./... - name: Run tests diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index b6625c01c..7ec43b611 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -15,16 +15,12 @@ jobs: with: fetch-depth: 0 - - name: "Set up Go" - uses: actions/setup-go@v6 - with: - go-version-file: go.mod + - uses: projectdiscovery/actions/setup/go@v1 - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v3 + uses: projectdiscovery/actions/goreleaser@v1 with: - args: "release --clean" - version: latest + release: true env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index f5b8b74c0..3cf46809e 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -16,13 +16,7 @@ jobs: with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - + - uses: projectdiscovery/actions/setup/go@v1 + - name: release test - uses: goreleaser/goreleaser-action@v4 - with: - args: "release --clean --snapshot" - version: latest + uses: projectdiscovery/actions/goreleaser@v1 From 8aa865375740fbbfd943a8aa90759b9837626064 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 23:24:42 +0100 Subject: [PATCH 128/132] chore(deps): bump github.com/cloudflare/circl from 1.6.1 to 1.6.3 (#1751) Bumps [github.com/cloudflare/circl](https://github.com/cloudflare/circl) from 1.6.1 to 1.6.3. - [Release notes](https://github.com/cloudflare/circl/releases) - [Commits](https://github.com/cloudflare/circl/compare/v1.6.1...v1.6.3) --- updated-dependencies: - dependency-name: github.com/cloudflare/circl dependency-version: 1.6.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5eb978410..aafa885c7 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.6.1 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/go.sum b/go.sum index 376070824..c60b30795 100644 --- a/go.sum +++ b/go.sum @@ -73,8 +73,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= From 2636330d223aa075112fa4c22c17eceae7330b71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 23:31:00 +0100 Subject: [PATCH 129/132] chore(deps): bump github.com/refraction-networking/utls (#1746) Bumps [github.com/refraction-networking/utls](https://github.com/refraction-networking/utls) from 1.7.1 to 1.8.2. - [Release notes](https://github.com/refraction-networking/utls/releases) - [Commits](https://github.com/refraction-networking/utls/compare/v1.7.1...v1.8.2) --- updated-dependencies: - dependency-name: github.com/refraction-networking/utls dependency-version: 1.8.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 +-- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index aafa885c7..e477dfc00 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,6 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.6.3 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -82,7 +81,7 @@ require ( github.com/projectdiscovery/hmap v0.0.98 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.31 // indirect - github.com/refraction-networking/utls v1.7.1 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect diff --git a/go.sum b/go.sum index c60b30795..ef69fe32c 100644 --- a/go.sum +++ b/go.sum @@ -73,8 +73,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= @@ -280,8 +278,8 @@ github.com/projectdiscovery/retryablehttp-go v1.1.0/go.mod h1:9DU57ezv5cfZSWw/m5 github.com/projectdiscovery/utils v0.7.3 h1:kX+77AA58yK6EZgkTRJEnK9V/7AZYzlXdcu/o/kJhFs= github.com/projectdiscovery/utils v0.7.3/go.mod h1:uDdQ3/VWomai98l+a3Ye/srDXdJ4xUIar/mSXlQ9gBM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= -github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= From bb5a68a23c5236d2cfacfe7ccecb658c7e73cd2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 23:38:36 +0100 Subject: [PATCH 130/132] chore(deps): bump the modules group with 10 updates (#1716) Bumps the modules group with 10 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) | `1.1.62` | `1.1.67` | | [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.82` | `0.0.83` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.1.0` | `1.1.1` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.7.3` | `0.8.0` | | [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) | `1.2.13` | `1.2.20` | | [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.4.19` | `0.5.2` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.98` | `0.0.99` | | [github.com/projectdiscovery/machineid](https://github.com/projectdiscovery/machineid) | `0.0.0-20240226150047-2e2c51e35983` | `0.0.0-20250715113114-c77eb3567582` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.31` | `0.1.33` | | [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) | `1.0.111` | `1.0.112` | Updates `github.com/projectdiscovery/gologger` from 1.1.62 to 1.1.67 - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.62...v1.1.67) Updates `github.com/projectdiscovery/ratelimit` from 0.0.82 to 0.0.83 - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.82...v0.0.83) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.1.0 to 1.1.1 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.1.0...v1.1.1) Updates `github.com/projectdiscovery/utils` from 0.7.3 to 0.8.0 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.7.3...v0.8.0) Updates `github.com/projectdiscovery/cdncheck` from 1.2.13 to 1.2.20 - [Release notes](https://github.com/projectdiscovery/cdncheck/releases) - [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.2.13...v1.2.20) Updates `github.com/projectdiscovery/fastdialer` from 0.4.19 to 0.5.2 - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.4.19...v0.5.2) Updates `github.com/projectdiscovery/hmap` from 0.0.98 to 0.0.99 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.98...v0.0.99) Updates `github.com/projectdiscovery/machineid` from 0.0.0-20240226150047-2e2c51e35983 to 0.0.0-20250715113114-c77eb3567582 - [Commits](https://github.com/projectdiscovery/machineid/commits) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.31 to 0.1.33 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.31...v0.1.33) Updates `github.com/projectdiscovery/retryabledns` from 1.0.111 to 1.0.112 - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.111...v1.0.112) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-version: 1.1.67 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/ratelimit dependency-version: 0.0.83 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-version: 1.1.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-version: 0.8.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/cdncheck dependency-version: 1.2.20 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/fastdialer dependency-version: 0.5.2 dependency-type: indirect update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-version: 0.0.99 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/machineid dependency-version: 0.0.0-20250715113114-c77eb3567582 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-version: 0.1.33 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryabledns dependency-version: 1.0.112 dependency-type: indirect update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 65 ++++++++++---------- go.sum | 185 ++++++++++++++++++++++++++++++++++----------------------- 2 files changed, 145 insertions(+), 105 deletions(-) diff --git a/go.mod b/go.mod index e477dfc00..a2e5d558c 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,10 @@ require ( github.com/projectdiscovery/chaos-client v0.5.2 github.com/projectdiscovery/dnsx v1.2.3 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.62 - github.com/projectdiscovery/ratelimit v0.0.82 - github.com/projectdiscovery/retryablehttp-go v1.1.0 - github.com/projectdiscovery/utils v0.7.3 + github.com/projectdiscovery/gologger v1.1.68 + github.com/projectdiscovery/ratelimit v0.0.83 + github.com/projectdiscovery/retryablehttp-go v1.3.2 + github.com/projectdiscovery/utils v0.9.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.11.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -28,22 +28,23 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect - github.com/STARRY-S/zip v0.2.1 // indirect + github.com/STARRY-S/zip v0.2.3 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect - github.com/andybalholm/brotli v1.1.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/sevenzip v1.6.1 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/djherbis/times v1.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect @@ -56,39 +57,40 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mholt/archives v0.1.0 // indirect + github.com/mholt/archives v0.1.5 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/mikelolasagasti/xz v1.0.1 // indirect + github.com/minio/minlz v1.0.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect - github.com/nwaples/rardecode/v2 v2.2.0 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.23 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.2.13 // indirect - github.com/projectdiscovery/fastdialer v0.4.19 // indirect - github.com/projectdiscovery/hmap v0.0.98 // indirect - github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.31 // indirect + github.com/projectdiscovery/cdncheck v1.2.27 // indirect + github.com/projectdiscovery/fastdialer v0.5.2 // indirect + github.com/projectdiscovery/hmap v0.0.99 // indirect + github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect + github.com/projectdiscovery/networkpolicy v0.1.33 // indirect github.com/refraction-networking/utls v1.8.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/sorairolake/lzip-go v0.3.8 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/therootcompany/xz v1.0.1 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect @@ -100,25 +102,24 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/ulikunitz/xz v0.5.15 // indirect - github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 // indirect + github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect + github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/mod v0.29.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.30.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.38.0 // indirect - gopkg.in/djherbis/times.v1 v1.3.0 // indirect + golang.org/x/tools v0.39.0 // indirect ) require ( @@ -131,7 +132,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.111 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect + github.com/projectdiscovery/retryabledns v1.0.113 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index ef69fe32c..01112048c 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,9 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= -github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= +github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= @@ -38,8 +39,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46 github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -54,10 +55,11 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWk github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= -github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= +github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= @@ -73,6 +75,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= @@ -84,6 +87,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -132,11 +137,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= -github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= +github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -156,11 +162,6 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -177,8 +178,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -194,6 +195,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -206,12 +209,16 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= -github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= +github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= +github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= +github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= +github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= +github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A= +github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -225,8 +232,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= -github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= -github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= +github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -237,8 +244,8 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= +github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -249,34 +256,34 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.2.13 h1:6zs4Mn8JV3yKyMoAr857Hf2NLvyOMpOfqCCT2V2OI1Q= -github.com/projectdiscovery/cdncheck v1.2.13/go.mod h1:/OhuZ9T25yXSqU6+oWvmVQ3QFvtew/Tp03u0jM+NJBE= +github.com/projectdiscovery/cdncheck v1.2.27 h1:nRi62iFIHQUZpOmsFYH8NuNX7A9Z0UFim6F7t1fEGyY= +github.com/projectdiscovery/cdncheck v1.2.27/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc= github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE= -github.com/projectdiscovery/fastdialer v0.4.19 h1:MLHwEGM0x0pyltJaNvAVvwc27bnXdZ5mYr50S/2kMEE= -github.com/projectdiscovery/fastdialer v0.4.19/go.mod h1:HGdVsez+JgJ9/ljXjHRplOqkB7og+nqi0nrNWVNi03o= +github.com/projectdiscovery/fastdialer v0.5.2 h1:BrK23yWc0XD57DMLqnF5oM5tBy8xx9brin+zoSo6gCw= +github.com/projectdiscovery/fastdialer v0.5.2/go.mod h1:euoxS1E93LDnl0OnNN0UALedAFF+EehBxyU3z+79l0g= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.62 h1:wzKqvL6HQRzf0/PpBEhInZqqL1q4mKe2gFGJeDG3FqE= -github.com/projectdiscovery/gologger v1.1.62/go.mod h1:YWvMSxlHybU3SkFCcWn+driSJ8yY+3CR3g/textnp+Y= -github.com/projectdiscovery/hmap v0.0.98 h1:XxYIi7yJCNiDAKCJXvuY9IBM5O6OgDgx4XHgKxkR4eg= -github.com/projectdiscovery/hmap v0.0.98/go.mod h1:bgN5fuZPJMj2YnAGEEnCypoifCnALJixHEVQszktQIU= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.31 h1:mE6iJeYOSql8gps/91vwiztE/kEHe5Im8oUO5Mkj9Zg= -github.com/projectdiscovery/networkpolicy v0.1.31/go.mod h1:5x4rGh4XhnoYl9wACnZyrjDGKIB/bQqxw2KrIM5V+XU= -github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak= -github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= -github.com/projectdiscovery/retryabledns v1.0.111 h1:iyMdCDgNmaSRJYcGqB+SLlvlw9WijlbJ6Q9OEpRAWsQ= -github.com/projectdiscovery/retryabledns v1.0.111/go.mod h1:6TOPJ3QAE4reBu6bvsGsTcyEb+OypcKYFQH7yVsjyIM= -github.com/projectdiscovery/retryablehttp-go v1.1.0 h1:uYp3EnuhhamTwvG41X6q6TAc/SHEO9pw9CBWbRASIQk= -github.com/projectdiscovery/retryablehttp-go v1.1.0/go.mod h1:9DU57ezv5cfZSWw/m5XFDTMjy1yKeMyn1kj35lPlcfM= -github.com/projectdiscovery/utils v0.7.3 h1:kX+77AA58yK6EZgkTRJEnK9V/7AZYzlXdcu/o/kJhFs= -github.com/projectdiscovery/utils v0.7.3/go.mod h1:uDdQ3/VWomai98l+a3Ye/srDXdJ4xUIar/mSXlQ9gBM= +github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU= +github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o= +github.com/projectdiscovery/hmap v0.0.99 h1:XPfLnD3CUrMqVCIdpK9ozD7Xmp3simx3T+2j4WWhHnU= +github.com/projectdiscovery/hmap v0.0.99/go.mod h1:koyUJi83K5G3w35ZLFXOYZIyYJsO+6hQrgDDN1RBrVE= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= +github.com/projectdiscovery/networkpolicy v0.1.33 h1:bVgp+XpLEsQ7ZEJt3UaUqIwhI01MMdt7F2dfIKFQg/w= +github.com/projectdiscovery/networkpolicy v0.1.33/go.mod h1:YAPddAXUc/lhoU85AFdvgOQKx8Qh8r0vzSjexRWk6Yk= +github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh8qHVROLQgl54= +github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= +github.com/projectdiscovery/retryabledns v1.0.113 h1:s+DAzdJ8XhLxRgt5636H0HG9OqHsGRjX9wTrLSTMqlQ= +github.com/projectdiscovery/retryabledns v1.0.113/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA= +github.com/projectdiscovery/retryablehttp-go v1.3.2 h1:Rv2gw/8t3QZz+WIuHUspVBoRrpBWpVOhzh/wLUGYSVM= +github.com/projectdiscovery/retryablehttp-go v1.3.2/go.mod h1:q1EQ+FX9JP5Z0EqLXDf+8b6XdzWmBXIMPowpI6hQ9aU= +github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= +github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= @@ -301,13 +308,17 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= -github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= +github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -320,8 +331,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= @@ -356,9 +365,9 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 h1:Bz/zVM/LoGZ9IztGBHrq2zlFQQbEG8dBYnxb4hamIHM= -github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39/go.mod h1:2oFzEwGYI7lhiqG0YkkcKa6VcpjVinQbWxaPzytDmLA= +github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko= +github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 h1:LiQSn5u8Nc6V/GixI+SWxt+YkNIyfKIlkVRULSw2Zt0= +github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= @@ -381,8 +390,8 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= @@ -403,13 +412,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -439,8 +450,11 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -463,14 +477,19 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -481,8 +500,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -506,7 +528,9 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -516,15 +540,25 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -535,8 +569,13 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -567,8 +606,10 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -609,13 +650,11 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= -gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= From 4d9e07828b8417b6664928d3028756890d559815 Mon Sep 17 00:00:00 2001 From: Pontus Lindblom Date: Fri, 20 Mar 2026 23:40:42 +0100 Subject: [PATCH 131/132] Remove dead facebook source (#1727) (#1732) The Meta CT API has been discontinued. --- pkg/passive/sources.go | 2 - pkg/passive/sources_test.go | 3 - pkg/subscraping/sources/facebook/ctlogs.go | 215 ------------------ .../sources/facebook/ctlogs_test.go | 71 ------ pkg/subscraping/sources/facebook/types.go | 36 --- 5 files changed, 327 deletions(-) delete mode 100644 pkg/subscraping/sources/facebook/ctlogs.go delete mode 100644 pkg/subscraping/sources/facebook/ctlogs_test.go delete mode 100644 pkg/subscraping/sources/facebook/types.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 79ec1eb4f..afcb46b21 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -28,7 +28,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" @@ -82,7 +81,6 @@ var AllSources = [...]subscraping.Source{ &dnsrepo.Source{}, &domainsproject.Source{}, &driftnet.Source{}, - &facebook.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 7c4c0da30..e86159335 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -54,7 +54,6 @@ var ( "windvane", "zoomeyeapi", "leakix", - "facebook", // "threatminer", // "reconcloud", "reconeer", @@ -99,7 +98,6 @@ var ( "virustotal", "whoisxmlapi", "leakix", - "facebook", // "threatminer", // "reconcloud", "reconeer", @@ -121,7 +119,6 @@ var ( "securitytrails", "virustotal", "leakix", - "facebook", "merklemap", "urlscan", // "reconcloud", diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go deleted file mode 100644 index dab71b8d7..000000000 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ /dev/null @@ -1,215 +0,0 @@ -package facebook - -import ( - "context" - "encoding/json" - "fmt" - "io" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - "github.com/projectdiscovery/utils/generic" - urlutil "github.com/projectdiscovery/utils/url" -) - -// source: https://developers.facebook.com/tools/ct -// api-docs: https://developers.facebook.com/docs/certificate-transparency-api -// ratelimit: ~20,000 req/hour per appID https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ - -var ( - domainsPerPage = "1000" - authUrl = "https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials" - domainsUrl = "https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=%s&limit=" + domainsPerPage -) - -type apiKey struct { - AppID string - Secret string - AccessToken string // obtained by calling - // https://graph.facebook.com/oauth/access_token?client_id=APP_ID&client_secret=APP_SECRET&grant_type=client_credentials - Error error // error while fetching access token -} - -// FetchAccessToken fetches the access token for the api key -// using app id and secret -func (k *apiKey) FetchAccessToken() { - if generic.EqualsAny("", k.AppID, k.Secret) { - k.Error = fmt.Errorf("invalid app id or secret") - return - } - resp, err := retryablehttp.Get(fmt.Sprintf(authUrl, k.AppID, k.Secret)) - if err != nil { - k.Error = err - return - } - defer func() { - if err := resp.Body.Close(); err != nil { - gologger.Error().Msgf("error closing response body: %s", err) - } - }() - bin, err := io.ReadAll(resp.Body) - if err != nil { - k.Error = err - return - } - auth := &authResponse{} - if err := json.Unmarshal(bin, auth); err != nil { - k.Error = err - return - } - if auth.AccessToken == "" { - k.Error = fmt.Errorf("invalid response from facebook got %v", string(bin)) - return - } - k.AccessToken = auth.AccessToken -} - -// IsValid returns true if the api key is valid -func (k *apiKey) IsValid() bool { - return k.AccessToken != "" -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []apiKey - timeTaken time.Duration - errors int - results int - requests int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - s.requests = 0 - - if len(s.apiKeys) == 0 { - s.skipped = true - close(results) - return results - } - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - key := subscraping.PickRandom(s.apiKeys, s.Name()) - domainsURL := fmt.Sprintf(domainsUrl, key.AccessToken, domain) - - for { - select { - case <-ctx.Done(): - return - default: - } - s.requests++ - resp, err := session.Get(ctx, domainsURL, "", nil) - if err != nil { - s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - bin, err := io.ReadAll(resp.Body) - if err != nil { - s.errors++ - gologger.Verbose().Msgf("failed to read response body: %s\n", err) - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - session.DiscardHTTPResponse(resp) - response := &response{} - if err := json.Unmarshal(bin, response); err != nil { - s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to unmarshal response: %s: %w", string(bin), err)} - return - } - for _, v := range response.Data { - for _, domain := range v.Domains { - select { - case <-ctx.Done(): - return - case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}: - s.results++ - } - } - } - if response.Paging.Next == "" { - break - } - domainsURL = updateParamInURL(response.Paging.Next, "limit", domainsPerPage) - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "facebook" -} - -// IsDefault returns true if the source should be queried by default -func (s *Source) IsDefault() bool { - return true -} - -// accepts subdomains (e.g. subdomain.domain.tld) -// but also returns all SANs for a certificate which may not match the domain -func (s *Source) HasRecursiveSupport() bool { - return true -} - -// KeyRequirement returns the API key requirement level for this source -func (s *Source) KeyRequirement() subscraping.KeyRequirement { - return subscraping.RequiredKey -} - -// NeedsKey returns true if the source requires an API key -func (s *Source) NeedsKey() bool { - return s.KeyRequirement() == subscraping.RequiredKey -} - -// AddApiKeys adds api keys to the source -func (s *Source) AddApiKeys(keys []string) { - allapikeys := subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - apiKey := apiKey{AppID: k, Secret: v} - apiKey.FetchAccessToken() - if apiKey.Error != nil { - gologger.Warning().Msgf("Could not fetch access token for %s: %s\n", k, apiKey.Error) - } - return apiKey - }) - // filter out invalid keys - for _, key := range allapikeys { - if key.IsValid() { - s.apiKeys = append(s.apiKeys, key) - } - } -} - -// Statistics returns the statistics for the source -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - Requests: s.requests, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} - -func updateParamInURL(url, param, value string) string { - urlx, err := urlutil.Parse(url) - if err != nil { - return url - } - urlx.Params.Set(param, value) - return urlx.String() -} diff --git a/pkg/subscraping/sources/facebook/ctlogs_test.go b/pkg/subscraping/sources/facebook/ctlogs_test.go deleted file mode 100644 index e9204055f..000000000 --- a/pkg/subscraping/sources/facebook/ctlogs_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package facebook - -import ( - "encoding/json" - "fmt" - "io" - "os" - "strings" - "testing" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/utils/generic" -) - -var ( - fb_API_ID = "$FB_APP_ID" - fb_API_SECRET = "$FB_APP_SECRET" -) - -func TestFacebookSource(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - updateWithEnv(&fb_API_ID) - updateWithEnv(&fb_API_SECRET) - if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) { - t.SkipNow() - } - k := apiKey{ - AppID: fb_API_ID, - Secret: fb_API_SECRET, - } - k.FetchAccessToken() - if k.Error != nil { - t.Fatal(k.Error) - } - - fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken) - resp, err := retryablehttp.Get(fetchURL) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := resp.Body.Close(); err != nil { - gologger.Error().Msgf("error closing response body: %s", err) - } - }() - bin, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - response := &response{} - if err := json.Unmarshal(bin, response); err != nil { - t.Fatal(err) - } - if len(response.Data) == 0 { - t.Fatal("no data found") - } -} - -func updateWithEnv(key *string) { - if key == nil { - return - } - value := *key - if strings.HasPrefix(value, "$") { - *key = os.Getenv(value[1:]) - } -} diff --git a/pkg/subscraping/sources/facebook/types.go b/pkg/subscraping/sources/facebook/types.go deleted file mode 100644 index 7a0fab23c..000000000 --- a/pkg/subscraping/sources/facebook/types.go +++ /dev/null @@ -1,36 +0,0 @@ -package facebook - -type authResponse struct { - AccessToken string `json:"access_token"` -} - -/* -{ - "data": [ - { - "domains": [ - "docs.hackerone.com" - ], - "id": "10056051421102939" - }, - ... - ], - "paging": { - "cursors": { - "before": "MTAwNTYwNTE0MjExMDI5MzkZD", - "after": "Njc0OTczNTA5NTA1MzUxNwZDZD" - }, - "next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD" - } -} -*/ -// example response - -type response struct { - Data []struct { - Domains []string `json:"domains"` - } `json:"data"` - Paging struct { - Next string `json:"next"` - } `json:"paging"` -} From e6ed1e1b72df7596a560b8b31b74354c4b0e2687 Mon Sep 17 00:00:00 2001 From: Pontus Lindblom Date: Tue, 24 Mar 2026 02:26:08 +0100 Subject: [PATCH 132/132] Chore - Code cleanup in workflows (#1768) * Change file extension to .yml for consistency * Move permissions for clarity and consistency * Change compat-checks action from @master to @v1 The action should be pinned to a stable ref such as @v1, consistent with the documentation and how other PD projects reference it. * Fix indentations and whitespaces in workflows --- .github/release.yml | 4 +- .github/workflows/codeql-analysis.yml | 31 +++++++------ .../{compat-checks.yaml => compat-checks.yml} | 7 +-- .github/workflows/release-binary.yml | 2 +- .github/workflows/release-test.yml | 2 +- .goreleaser.yml | 46 +++++++++---------- 6 files changed, 47 insertions(+), 45 deletions(-) rename .github/workflows/{compat-checks.yaml => compat-checks.yml} (73%) diff --git a/.github/release.yml b/.github/release.yml index 1073bdc8a..550dbea54 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -8,10 +8,10 @@ changelog: - "Type: Enhancement" - title: 🐞 Bug Fixes labels: - - "Type: Bug" + - "Type: Bug" - title: πŸ”¨ Maintenance labels: - - "Type: Maintenance" + - "Type: Maintenance" - title: Other Changes labels: - "*" \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 31794f2a7..516d067b4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,14 +7,15 @@ on: - '**.go' - '**.mod' +permissions: + actions: read + contents: read + security-events: write + jobs: analyze: name: Analyze runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write strategy: fail-fast: false @@ -23,17 +24,17 @@ jobs: # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] steps: - - name: Checkout repository - uses: actions/checkout@v6 + - name: Checkout repository + uses: actions/checkout@v6 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} - - name: Autobuild - uses: github/codeql-action/autobuild@v4 + - name: Autobuild + uses: github/codeql-action/autobuild@v4 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/compat-checks.yaml b/.github/workflows/compat-checks.yml similarity index 73% rename from .github/workflows/compat-checks.yaml rename to .github/workflows/compat-checks.yml index a19241221..7f6e9bc8b 100644 --- a/.github/workflows/compat-checks.yaml +++ b/.github/workflows/compat-checks.yml @@ -6,14 +6,15 @@ on: branches: - dev +permissions: + contents: write + jobs: check: if: github.actor == 'dependabot[bot]' runs-on: ubuntu-latest - permissions: - contents: write steps: - uses: actions/checkout@v6 - - uses: projectdiscovery/actions/setup/go/compat-checks@master + - uses: projectdiscovery/actions/setup/go/compat-checks@v1 with: go-version-file: go.mod diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 7ec43b611..55f340d03 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -12,7 +12,7 @@ jobs: steps: - name: "Check out code" uses: actions/checkout@v6 - with: + with: fetch-depth: 0 - uses: projectdiscovery/actions/setup/go@v1 diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 3cf46809e..6c5b6cc99 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -13,7 +13,7 @@ jobs: steps: - name: "Check out code" uses: actions/checkout@v6 - with: + with: fetch-depth: 0 - uses: projectdiscovery/actions/setup/go@v1 diff --git a/.goreleaser.yml b/.goreleaser.yml index e34562f04..151920f61 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -5,31 +5,31 @@ before: - go mod tidy builds: -- env: - - CGO_ENABLED=0 - goos: - - windows - - linux - - darwin - goarch: - - amd64 - - '386' - - arm - - arm64 - - ignore: - - goos: darwin - goarch: '386' - - goos: windows - goarch: 'arm' - - binary: '{{ .ProjectName }}' - main: cmd/subfinder/main.go + - env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - '386' + - arm + - arm64 + + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' + + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go archives: -- formats: - - zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' + - formats: + - zip + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: algorithm: sha256