diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..ff63af57f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Proposed changes + + + +### Proof + + + +## Checklist + + + +- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/subfinder/tree/dev) branch +- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added necessary documentation (if appropriate) \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4ff8f26cb..63f4b632f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,17 +8,20 @@ updates: # Maintain dependencies for go modules - package-ecosystem: "gomod" - directory: "v2/" + directory: "/" schedule: interval: "weekly" target-branch: "dev" commit-message: prefix: "chore" include: "scope" - labels: - - "Type: Maintenance" allow: - dependency-name: "github.com/projectdiscovery/*" + groups: + modules: + patterns: ["github.com/projectdiscovery/*"] + labels: + - "Type: Maintenance" # # Maintain dependencies for GitHub Actions # - package-ecosystem: "github-actions" @@ -42,4 +45,4 @@ updates: # prefix: "chore" # include: "scope" # labels: -# - "Type: Maintenance" \ No newline at end of file +# - "Type: Maintenance" diff --git a/.github/release.yml b/.github/release.yml index 1073bdc8a..550dbea54 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -8,10 +8,10 @@ changelog: - "Type: Enhancement" - title: 🐞 Bug Fixes labels: - - "Type: Bug" + - "Type: Bug" - title: πŸ”¨ Maintenance labels: - - "Type: Maintenance" + - "Type: Maintenance" - title: Other Changes labels: - "*" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 02e1df74a..939fc835d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -3,35 +3,37 @@ name: πŸ”¨ Build Test on: pull_request: paths: - - '**.go' - - '**.mod' + - "**.go" + - "**.mod" workflow_dispatch: inputs: short: - description: 'Use -short flag for tests' + description: "Use -short flag for tests" required: false type: boolean default: false -jobs: +jobs: + lint: + name: Lint Test + if: "${{ !endsWith(github.actor, '[bot]') }}" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/actions/golangci-lint/v2@v1 + build: name: Test Builds + needs: [lint] runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-13] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - - name: Check out code - uses: actions/checkout@v3 - - - name: Build - run: go build ./... - working-directory: v2/ + - uses: actions/checkout@v6 + - uses: projectdiscovery/actions/setup/go@v1 + - run: go build ./... - name: Run tests env: @@ -48,7 +50,6 @@ jobs: FOFA_API_KEY: ${{secrets.FOFA_API_KEY}} FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}} GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}} - HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}} INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} @@ -56,19 +57,19 @@ jobs: SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} + URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v3 with: timeout_seconds: 360 max_attempts: 3 - command: cd v2; go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} + command: go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} - name: Race Condition Tests run: go build -race ./... - working-directory: v2/ - name: Run Example run: go run . - working-directory: v2/examples \ No newline at end of file + working-directory: examples diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 927b1cc11..516d067b4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,14 +7,15 @@ on: - '**.go' - '**.mod' +permissions: + actions: read + contents: read + security-events: write + jobs: analyze: name: Analyze runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write strategy: fail-fast: false @@ -23,17 +24,17 @@ jobs: # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v6 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} - - name: Autobuild - uses: github/codeql-action/autobuild@v2 + - name: Autobuild + uses: github/codeql-action/autobuild@v4 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/compat-checks.yml b/.github/workflows/compat-checks.yml new file mode 100644 index 000000000..7f6e9bc8b --- /dev/null +++ b/.github/workflows/compat-checks.yml @@ -0,0 +1,20 @@ +name: ♾️ Compatibility Checks + +on: + pull_request: + types: [opened, synchronize] + branches: + - dev + +permissions: + contents: write + +jobs: + check: + if: github.actor == 'dependabot[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: projectdiscovery/actions/setup/go/compat-checks@v1 + with: + go-version-file: go.mod diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml index 84b26e1fe..a21b93389 100644 --- a/.github/workflows/dep-auto-merge.yml +++ b/.github/workflows/dep-auto-merge.yml @@ -16,11 +16,11 @@ jobs: runs-on: ubuntu-latest if: github.actor == 'dependabot[bot]' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: token: ${{ secrets.DEPENDABOT_PAT }} - uses: ahmadnassri/action-dependabot-auto-merge@v2 with: github-token: ${{ secrets.DEPENDABOT_PAT }} - target: all \ No newline at end of file + target: all diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 8c328ce4b..582c32535 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Get Github tag id: meta @@ -20,21 +20,21 @@ jobs: curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} \ No newline at end of file + tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml deleted file mode 100644 index da4f58218..000000000 --- a/.github/workflows/lint-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: πŸ™πŸ» Lint Test - -on: - pull_request: - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -jobs: - lint: - name: Lint Test - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.6.0 - with: - version: latest - args: --timeout 5m - working-directory: v2/ \ No newline at end of file diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 487b3f5d3..55f340d03 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -11,23 +11,18 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 - with: + uses: actions/checkout@v6 + with: fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - + + - uses: projectdiscovery/actions/setup/go@v1 + - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v3 + uses: projectdiscovery/actions/goreleaser@v1 with: - args: "release --clean" - version: latest - workdir: v2/ + release: true env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index d5b099447..6c5b6cc99 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -12,18 +12,11 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 - with: + uses: actions/checkout@v6 + with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - + - uses: projectdiscovery/actions/setup/go@v1 + - name: release test - uses: goreleaser/goreleaser-action@v4 - with: - args: "release --clean --snapshot" - version: latest - workdir: v2 \ No newline at end of file + uses: projectdiscovery/actions/goreleaser@v1 diff --git a/.gitignore b/.gitignore index b0dd47ba3..155676bfe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ vendor/ .idea .devcontainer .vscode -dist \ No newline at end of file +dist +/subfinder \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 000000000..151920f61 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,46 @@ +version: 2 + +before: + hooks: + - go mod tidy + +builds: + - env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - '386' + - arm + - arm64 + + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' + + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go + +archives: + - formats: + - zip + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' + +checksum: + algorithm: sha256 + +announce: + slack: + enabled: true + channel: '#release' + username: GoReleaser + message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' + + discord: + enabled: true + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' diff --git a/Dockerfile b/Dockerfile index eadf4f7e1..025cab042 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,15 @@ # Build -FROM golang:1.21-alpine AS build-env +FROM golang:1.24-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app -WORKDIR /app/v2 RUN go mod download RUN go build ./cmd/subfinder # Release -FROM alpine:3.18.6 +FROM alpine:latest RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates -COPY --from=build-env /app/v2/subfinder /usr/local/bin/ +COPY --from=build-env /app/subfinder /usr/local/bin/ ENTRYPOINT ["subfinder"] diff --git a/v2/Makefile b/Makefile similarity index 100% rename from v2/Makefile rename to Makefile diff --git a/README.md b/README.md index 97e0b09fa..dad545074 100644 --- a/README.md +++ b/README.md @@ -109,9 +109,16 @@ OPTIMIZATION: -max-time int minutes to wait for enumeration results (default 10) ``` +## Environment Variables + +Subfinder supports environment variables to specify custom paths for configuration files: + +- `SUBFINDER_CONFIG` - Path to config.yaml file (overrides default `$CONFIG/subfinder/config.yaml`) +- `SUBFINDER_PROVIDER_CONFIG` - Path to provider-config.yaml file (overrides default `$CONFIG/subfinder/provider-config.yaml`) + # Installation -`subfinder` requires **go1.21** to install successfully. Run the following command to install the latest version: +`subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest @@ -129,7 +136,7 @@ Learn about how to run Subfinder here: https://docs.projectdiscovery.io/tools/su ## Subfinder Go library -Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](v2/examples/main.go) +Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](examples/main.go) diff --git a/v2/cmd/subfinder/main.go b/cmd/subfinder/main.go similarity index 100% rename from v2/cmd/subfinder/main.go rename to cmd/subfinder/main.go diff --git a/v2/examples/main.go b/examples/main.go similarity index 100% rename from v2/examples/main.go rename to examples/main.go diff --git a/v2/go.mod b/go.mod similarity index 59% rename from v2/go.mod rename to go.mod index cac91726d..a2e5d558c 100644 --- a/v2/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/projectdiscovery/subfinder/v2 -go 1.21 +go 1.24.0 + +toolchain go1.24.1 require ( github.com/corpix/uarand v0.2.0 @@ -8,16 +10,16 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.2 - github.com/projectdiscovery/dnsx v1.2.1 + github.com/projectdiscovery/dnsx v1.2.3 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.43 - github.com/projectdiscovery/ratelimit v0.0.70 - github.com/projectdiscovery/retryablehttp-go v1.0.97 - github.com/projectdiscovery/utils v0.4.9 + github.com/projectdiscovery/gologger v1.1.68 + github.com/projectdiscovery/ratelimit v0.0.83 + github.com/projectdiscovery/retryablehttp-go v1.3.2 + github.com/projectdiscovery/utils v0.9.0 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.11.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 - golang.org/x/exp v0.0.0-20230420155640-133eef4313cb + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 gopkg.in/yaml.v3 v3.0.1 ) @@ -26,64 +28,72 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/STARRY-S/zip v0.2.3 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect - github.com/andybalholm/brotli v1.0.6 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.1 // indirect + github.com/bodgit/windows v1.0.1 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.3.7 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/djherbis/times v1.6.0 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaissmai/bart v0.9.5 // indirect + github.com/gaissmai/bart v0.26.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mholt/archiver/v3 v3.5.1 // indirect + github.com/mholt/archives v0.1.5 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/mikelolasagasti/xz v1.0.1 // indirect + github.com/minio/minlz v1.0.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect - github.com/nwaples/rardecode v1.1.3 // indirect - github.com/pierrec/lz4/v4 v4.1.2 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.23 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.0 // indirect - github.com/projectdiscovery/fastdialer v0.3.0 // indirect - github.com/projectdiscovery/hmap v0.0.78 // indirect - github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.1 // indirect - github.com/refraction-networking/utls v1.6.7 // indirect + github.com/projectdiscovery/cdncheck v1.2.27 // indirect + github.com/projectdiscovery/fastdialer v0.5.2 // indirect + github.com/projectdiscovery/hmap v0.0.99 // indirect + github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect + github.com/projectdiscovery/networkpolicy v0.1.33 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sorairolake/lzip-go v0.3.8 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -91,41 +101,38 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect - github.com/weppos/publicsuffix-go v0.30.1 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect + github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/djherbis/times.v1 v1.3.0 // indirect + golang.org/x/tools v0.39.0 // indirect ) require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/miekg/dns v1.1.56 // indirect + github.com/miekg/dns v1.1.62 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.64 - github.com/projectdiscovery/retryabledns v1.0.94 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/projectdiscovery/goflags v0.1.74 + github.com/projectdiscovery/retryabledns v1.0.113 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect ) diff --git a/v2/go.sum b/go.sum similarity index 50% rename from v2/go.sum rename to go.sum index 4c9ece9b8..01112048c 100644 --- a/v2/go.sum +++ b/go.sum @@ -1,6 +1,25 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= @@ -8,6 +27,8 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2piv github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= +github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= @@ -18,9 +39,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46 github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -33,7 +53,14 @@ github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJR github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= +github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= +github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= +github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= +github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= +github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= +github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= @@ -44,85 +71,118 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99k github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= -github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= +github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0= +github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= -github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= -github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= -github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -135,6 +195,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -147,12 +209,16 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= -github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= +github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= +github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= -github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= +github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= +github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= +github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A= +github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -166,9 +232,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= -github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= +github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -179,55 +244,60 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= -github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= +github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= -github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= +github.com/projectdiscovery/cdncheck v1.2.27 h1:nRi62iFIHQUZpOmsFYH8NuNX7A9Z0UFim6F7t1fEGyY= +github.com/projectdiscovery/cdncheck v1.2.27/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= -github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg= -github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0= -github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk= -github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= +github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc= +github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE= +github.com/projectdiscovery/fastdialer v0.5.2 h1:BrK23yWc0XD57DMLqnF5oM5tBy8xx9brin+zoSo6gCw= +github.com/projectdiscovery/fastdialer v0.5.2/go.mod h1:euoxS1E93LDnl0OnNN0UALedAFF+EehBxyU3z+79l0g= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= -github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= -github.com/projectdiscovery/gologger v1.1.43 h1:26DOeBUK2xus/UpM8jzHfNqEU5tWams3VGBtjJtI02I= -github.com/projectdiscovery/gologger v1.1.43/go.mod h1:993FxohnjVo34dSgE3bw+L4TOCDNQfQ5zNbK0YhYrEw= -github.com/projectdiscovery/hmap v0.0.78 h1:eUjLjFR7KaxnlSIVQgT/Uc+i3EULGFb9Ax8qYAbbZno= -github.com/projectdiscovery/hmap v0.0.78/go.mod h1:5iJ3+EtjRuechPw0W/9Mq5IDIMh68IBcIBEoLqS20NM= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= -github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU= -github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0vd8ER+eCHQjc= -github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM= -github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= -github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= -github.com/projectdiscovery/retryablehttp-go v1.0.97 h1:6nee/vJjiZP3vOhyqLcpSADM3vqmcC2QOvaMIo+dKWQ= -github.com/projectdiscovery/retryablehttp-go v1.0.97/go.mod h1:ZvwB6IsIHf0YlovcEQufZ6OTluyWfxRd360SrKd9fPk= -github.com/projectdiscovery/utils v0.4.9 h1:GzYKy5iiCWEZZPGxrtgTOnRTZYiIAiCditGufp0nhGU= -github.com/projectdiscovery/utils v0.4.9/go.mod h1:/68d0OHGgYF4aW4X7kS1qlFlYOnZxgtFDN85iH732JI= -github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= -github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= +github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= +github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= +github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU= +github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o= +github.com/projectdiscovery/hmap v0.0.99 h1:XPfLnD3CUrMqVCIdpK9ozD7Xmp3simx3T+2j4WWhHnU= +github.com/projectdiscovery/hmap v0.0.99/go.mod h1:koyUJi83K5G3w35ZLFXOYZIyYJsO+6hQrgDDN1RBrVE= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= +github.com/projectdiscovery/networkpolicy v0.1.33 h1:bVgp+XpLEsQ7ZEJt3UaUqIwhI01MMdt7F2dfIKFQg/w= +github.com/projectdiscovery/networkpolicy v0.1.33/go.mod h1:YAPddAXUc/lhoU85AFdvgOQKx8Qh8r0vzSjexRWk6Yk= +github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh8qHVROLQgl54= +github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= +github.com/projectdiscovery/retryabledns v1.0.113 h1:s+DAzdJ8XhLxRgt5636H0HG9OqHsGRjX9wTrLSTMqlQ= +github.com/projectdiscovery/retryabledns v1.0.113/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA= +github.com/projectdiscovery/retryablehttp-go v1.3.2 h1:Rv2gw/8t3QZz+WIuHUspVBoRrpBWpVOhzh/wLUGYSVM= +github.com/projectdiscovery/retryablehttp-go v1.3.2/go.mod h1:q1EQ+FX9JP5Z0EqLXDf+8b6XdzWmBXIMPowpI6hQ9aU= +github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8= +github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= @@ -238,20 +308,27 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= +github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= @@ -261,8 +338,8 @@ github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EU github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -285,15 +362,14 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= -github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko= +github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 h1:LiQSn5u8Nc6V/GixI+SWxt+YkNIyfKIlkVRULSw2Zt0= +github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -314,66 +390,137 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -383,6 +530,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -392,18 +540,28 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -412,34 +570,92 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= -gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -448,3 +664,11 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/v2/pkg/passive/doc.go b/pkg/passive/doc.go similarity index 100% rename from v2/pkg/passive/doc.go rename to pkg/passive/doc.go diff --git a/v2/pkg/passive/passive.go b/pkg/passive/passive.go similarity index 72% rename from v2/pkg/passive/passive.go rename to pkg/passive/passive.go index 199686bd4..c00a024d5 100644 --- a/v2/pkg/passive/passive.go +++ b/pkg/passive/passive.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + mapsutil "github.com/projectdiscovery/utils/maps" ) type EnumerationOptions struct { @@ -65,11 +66,15 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p for _, runner := range a.sources { wg.Add(1) go func(source subscraping.Source) { + defer wg.Done() ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { - results <- resp + select { + case <-ctx.Done(): + return + case results <- resp: + } } - wg.Done() }(runner) } wg.Wait() @@ -81,14 +86,21 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error - for _, source := range a.sources { - var rl uint - if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { - rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit) + if rateLimit == nil { + rateLimit = &subscraping.CustomRateLimit{ + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, } + } + for _, source := range a.sources { + rl, duration := resolveSourceRateLimit(globalRateLimit, rateLimit, source.Name()) if rl > 0 { - multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second) + multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, duration) } else { multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond) } @@ -100,6 +112,30 @@ func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, return multiRateLimiter, err } +// resolveSourceRateLimit returns the effective rate limit and duration for a source. +// Priority: per-source custom limit > global -rl limit > unlimited. +// Duration comes from -rls (e.g. hackertarget=2/m β†’ 2 per minute), defaulting to per-second. +func resolveSourceRateLimit(globalRateLimit int, rateLimit *subscraping.CustomRateLimit, sourceName string) (uint, time.Duration) { + duration := time.Second // default: requests per second + + sourceLower := strings.ToLower(sourceName) + if sourceRL, ok := rateLimit.Custom.Get(sourceLower); ok { + rl := sourceRateLimitOrDefault(uint(max(globalRateLimit, 0)), sourceRL) + // Use per-source duration from -rls if set (e.g. "2/m" β†’ time.Minute) + if d, ok := rateLimit.CustomDuration.Get(sourceLower); ok && d > 0 { + duration = d + } + return rl, duration + } + + // No per-source limit: fall back to global -rl + if globalRateLimit > 0 { + return uint(globalRateLimit), duration + } + + return 0, duration +} + func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint { if sourceRateLimit > 0 { return sourceRateLimit diff --git a/pkg/passive/ratelimit_test.go b/pkg/passive/ratelimit_test.go new file mode 100644 index 000000000..36c929e1e --- /dev/null +++ b/pkg/passive/ratelimit_test.go @@ -0,0 +1,147 @@ +package passive + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mapsutil "github.com/projectdiscovery/utils/maps" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +func newCustomRateLimit(counts map[string]uint, durations map[string]time.Duration) *subscraping.CustomRateLimit { + crl := &subscraping.CustomRateLimit{ + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, + } + for k, v := range counts { + _ = crl.Custom.Set(k, v) + } + for k, d := range durations { + _ = crl.CustomDuration.Set(k, d) + } + return crl +} + +func TestResolveSourceRateLimit_PerSourceOverride(t *testing.T) { + // Per-source limit should override global + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 5}, + map[string]time.Duration{"hackertarget": time.Minute}, + ) + + rl, dur := resolveSourceRateLimit(10, crl, "hackertarget") + assert.Equal(t, uint(5), rl) + assert.Equal(t, time.Minute, dur) +} + +func TestResolveSourceRateLimit_GlobalFallback(t *testing.T) { + // Sources without per-source limit should use global -rl + crl := newCustomRateLimit(nil, nil) + + rl, dur := resolveSourceRateLimit(10, crl, "crtsh") + assert.Equal(t, uint(10), rl) + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_NoLimit(t *testing.T) { + // No global and no per-source β†’ unlimited (0) + crl := newCustomRateLimit(nil, nil) + + rl, _ := resolveSourceRateLimit(0, crl, "crtsh") + assert.Equal(t, uint(0), rl) +} + +func TestResolveSourceRateLimit_ZeroValueRateLimit(t *testing.T) { + // Zero-value CustomRateLimit (nil maps) should not panic and fall back to global + rl, dur := resolveSourceRateLimit(5, &subscraping.CustomRateLimit{}, "crtsh") + assert.Equal(t, uint(5), rl) + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_PerSourceDefaultDuration(t *testing.T) { + // Per-source limit without explicit duration defaults to per-second + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 3}, + nil, + ) + + rl, dur := resolveSourceRateLimit(0, crl, "hackertarget") + assert.Equal(t, uint(3), rl) + assert.Equal(t, time.Second, dur) +} + +func TestBuildMultiRateLimiter_GlobalAppliedToAll(t *testing.T) { + // With -rl 5 and no per-source overrides, all sources should be rate-limited + agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false) + crl := newCustomRateLimit(nil, nil) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 5, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_NilRateLimit(t *testing.T) { + // nil rateLimit should not panic (guard in buildMultiRateLimiter) + agent := New([]string{"hackertarget"}, []string{}, false, false) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, nil) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_PerSourceWithDuration(t *testing.T) { + // Per-source limit with custom duration (e.g. -rls hackertarget=2/m) + agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false) + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 2}, + map[string]time.Duration{"hackertarget": time.Minute}, + ) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_UnlimitedWhenNoLimits(t *testing.T) { + // Without any rate limits, sources should get MaxUint32 (effectively unlimited) + agent := New([]string{"hackertarget"}, []string{}, false, false) + crl := newCustomRateLimit(nil, nil) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestResolveSourceRateLimit_PerSourceExceedsGlobal(t *testing.T) { + // Per-source limit higher than global should be honoured + // (the source was explicitly allowed more than the default) + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 20}, + nil, + ) + + rl, dur := resolveSourceRateLimit(10, crl, "hackertarget") + assert.Equal(t, uint(20), rl, "per-source limit should take precedence even when > global") + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_CaseInsensitive(t *testing.T) { + // Source names should be case-insensitive for lookup + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 5}, + nil, + ) + + // Try mixed case β€” should still match the lower-cased key + rl, _ := resolveSourceRateLimit(0, crl, "HackerTarget") + assert.Equal(t, uint(5), rl, "source name lookup should be case-insensitive") +} diff --git a/v2/pkg/passive/sources.go b/pkg/passive/sources.go similarity index 81% rename from v2/pkg/passive/sources.go rename to pkg/passive/sources.go index 3b581b594..afcb46b21 100644 --- a/v2/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -1,6 +1,8 @@ package passive import ( + "fmt" + "os" "strings" "golang.org/x/exp/maps" @@ -10,7 +12,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/builtwith" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" @@ -20,31 +21,42 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -53,8 +65,8 @@ var AllSources = [...]subscraping.Source{ &alienvault.Source{}, &anubis.Source{}, &bevigil.Source{}, - &binaryedge.Source{}, &bufferover.Source{}, + &builtwith.Source{}, &c99.Source{}, &censys.Source{}, &certspotter.Source{}, @@ -62,36 +74,46 @@ var AllSources = [...]subscraping.Source{ &chinaz.Source{}, &commoncrawl.Source{}, &crtsh.Source{}, + &digitalyama.Source{}, &digitorus.Source{}, &dnsdb.Source{}, &dnsdumpster.Source{}, &dnsrepo.Source{}, + &domainsproject.Source{}, + &driftnet.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, &hackertarget.Source{}, - &hunter.Source{}, + &hudsonrock.Source{}, &intelx.Source{}, - &netlas.Source{}, &leakix.Source{}, + &merklemap.Source{}, + &netlas.Source{}, + &onyphe.Source{}, + &profundis.Source{}, + &pugrecon.Source{}, &quake.Source{}, &rapiddns.Source{}, + // &reconcloud.Source{}, // failing due to cloudflare bot protection + &reconeer.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, + &rsecloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, + &thc.Source{}, &threatbook.Source{}, + &threatcrowd.Source{}, + // &threatminer.Source{}, // failing api + &urlscan.Source{}, &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, + &windvane.Source{}, &zoomeyeapi.Source{}, - &facebook.Source{}, - // &threatminer.Source{}, // failing api - // &reconcloud.Source{}, // failing due to cloudflare bot protection - &builtwith.Source{}, - &hudsonrock.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( @@ -162,6 +184,15 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup } } + for _, source := range sources { + keyReq := source.KeyRequirement() + if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { + if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" { + source.AddApiKeys([]string{apiKey}) + } + } + } + // Create the agent, insert the sources and remove the excluded sources agent := &Agent{sources: maps.Values(sources)} diff --git a/v2/pkg/passive/sources_test.go b/pkg/passive/sources_test.go similarity index 90% rename from v2/pkg/passive/sources_test.go rename to pkg/passive/sources_test.go index 916e6c82a..e86159335 100644 --- a/v2/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -14,7 +14,6 @@ var ( "alienvault", "anubis", "bevigil", - "binaryedge", "bufferover", "c99", "censys", @@ -27,32 +26,43 @@ var ( "dnsdumpster", "dnsdb", "dnsrepo", + "domainsproject", + "driftnet", "fofa", "fullhunt", "github", "hackertarget", "intelx", "netlas", + "onyphe", "quake", + "pugrecon", "rapiddns", "redhuntlabs", // "riddler", // failing due to cloudfront protection "robtex", + "rsecloud", "securitytrails", + "profundis", "shodan", "sitedossier", "threatbook", + "threatcrowd", "virustotal", "waybackarchive", "whoisxmlapi", + "windvane", "zoomeyeapi", - "hunter", "leakix", - "facebook", // "threatminer", // "reconcloud", + "reconeer", "builtwith", "hudsonrock", + "digitalyama", + "merklemap", + "thc", + "urlscan", } expectedDefaultSources = []string{ @@ -68,40 +78,49 @@ var ( "crtsh", "digitorus", "dnsdumpster", + "domainsproject", "dnsrepo", + "driftnet", "fofa", "fullhunt", "hackertarget", "intelx", + "onyphe", "quake", "redhuntlabs", "robtex", // "riddler", // failing due to cloudfront protection + "rsecloud", "securitytrails", + "profundis", "shodan", + "windvane", "virustotal", "whoisxmlapi", - "hunter", "leakix", - "facebook", // "threatminer", // "reconcloud", + "reconeer", "builtwith", + "digitalyama", + "thc", + "urlscan", } expectedDefaultRecursiveSources = []string{ "alienvault", - "binaryedge", "bufferover", "certspotter", "crtsh", "dnsdb", "digitorus", + "driftnet", "hackertarget", "securitytrails", "virustotal", "leakix", - "facebook", + "merklemap", + "urlscan", // "reconcloud", } ) diff --git a/v2/pkg/passive/sources_w_auth_test.go b/pkg/passive/sources_w_auth_test.go similarity index 100% rename from v2/pkg/passive/sources_w_auth_test.go rename to pkg/passive/sources_w_auth_test.go diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go similarity index 68% rename from v2/pkg/passive/sources_wo_auth_test.go rename to pkg/passive/sources_wo_auth_test.go index d5aef6530..c3316d63c 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -24,14 +24,19 @@ func TestSourcesWithoutKeys(t *testing.T) { } ignoredSources := []string{ - "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - "riddler", // failing due to cloudfront protection - "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. - "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally - "waybackarchive", // Fails randomly - "alienvault", // 503 Service Temporarily Unavailable - "digitorus", // failing with "Failed to retrieve certificate" - "dnsdumpster", // failing with "unexpected status code 403 received" + "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ + "riddler", // failing due to cloudfront protection + "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally + "waybackarchive", // Fails randomly + "alienvault", // 503 Service Temporarily Unavailable + "digitorus", // failing with "Failed to retrieve certificate" + "dnsdumpster", // failing with "unexpected status code 403 received" + "anubis", // failing with "too many redirects" + "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" + "leakix", // now requires API key (returns 401) + "reconeer", // now requires API key (returns 401) + "sitedossier", // flaky - returns no results in CI } domain := "hackerone.com" diff --git a/v2/pkg/resolve/client.go b/pkg/resolve/client.go similarity index 100% rename from v2/pkg/resolve/client.go rename to pkg/resolve/client.go diff --git a/v2/pkg/resolve/doc.go b/pkg/resolve/doc.go similarity index 100% rename from v2/pkg/resolve/doc.go rename to pkg/resolve/doc.go diff --git a/v2/pkg/resolve/resolve.go b/pkg/resolve/resolve.go similarity index 81% rename from v2/pkg/resolve/resolve.go rename to pkg/resolve/resolve.go index 271d699fb..38055b89b 100644 --- a/v2/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -25,18 +25,20 @@ type ResolutionPool struct { // HostEntry defines a host with the source type HostEntry struct { - Domain string - Host string - Source string + Domain string + Host string + Source string + WildcardCertificate bool } // Result contains the result for a host resolution type Result struct { - Type ResultType - Host string - IP string - Error error - Source string + Type ResultType + Host string + IP string + Error error + Source string + WildcardCertificate bool } // ResultType is the type of result found @@ -60,7 +62,7 @@ func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *Resoluti } go func() { - for i := 0; i < workers; i++ { + for range workers { resolutionPool.wg.Add(1) go resolutionPool.resolveWorker() } @@ -73,7 +75,7 @@ func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *Resoluti // InitWildcards inits the wildcard ips array func (r *ResolutionPool) InitWildcards(domain string) error { - for i := 0; i < maxWildcardChecks; i++ { + for range maxWildcardChecks { uid := xid.New().String() hosts, _ := r.DNSClient.Lookup(uid + "." + domain) @@ -92,13 +94,13 @@ func (r *ResolutionPool) InitWildcards(domain string) error { func (r *ResolutionPool) resolveWorker() { for task := range r.Tasks { if !r.removeWildcard { - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source, WildcardCertificate: task.WildcardCertificate} continue } hosts, err := r.DNSClient.Lookup(task.Host) if err != nil { - r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} + r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err, WildcardCertificate: task.WildcardCertificate} continue } @@ -116,7 +118,7 @@ func (r *ResolutionPool) resolveWorker() { } if !skip { - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source, WildcardCertificate: task.WildcardCertificate} } } r.wg.Done() diff --git a/v2/pkg/runner/banners.go b/pkg/runner/banners.go similarity index 97% rename from v2/pkg/runner/banners.go rename to pkg/runner/banners.go index 1e9af43bc..a96fa70db 100644 --- a/v2/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.6.8` +const version = `v2.13.0` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/pkg/runner/config.go b/pkg/runner/config.go similarity index 79% rename from v2/pkg/runner/config.go rename to pkg/runner/config.go index 6c8ed2c6f..b0c65205f 100644 --- a/v2/pkg/runner/config.go +++ b/pkg/runner/config.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" fileutil "github.com/projectdiscovery/utils/file" ) @@ -17,11 +18,16 @@ func createProviderConfigYAML(configFilePath string) error { if err != nil { return err } - defer configFile.Close() + defer func() { + if err := configFile.Close(); err != nil { + gologger.Error().Msgf("Error closing config file: %s", err) + } + }() sourcesRequiringApiKeysMap := make(map[string][]string) for _, source := range passive.AllSources { - if source.NeedsKey() { + keyReq := source.KeyRequirement() + if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { sourceName := strings.ToLower(source.Name()) sourcesRequiringApiKeysMap[sourceName] = []string{} } @@ -42,7 +48,7 @@ func UnmarshalFrom(file string) error { for _, source := range passive.AllSources { sourceName := strings.ToLower(source.Name()) apiKeys := sourceApiKeysMap[sourceName] - if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 { + if len(apiKeys) > 0 { gologger.Debug().Msgf("API key(s) found for %s.", sourceName) source.AddApiKeys(apiKeys) } diff --git a/v2/pkg/runner/doc.go b/pkg/runner/doc.go similarity index 100% rename from v2/pkg/runner/doc.go rename to pkg/runner/doc.go diff --git a/v2/pkg/runner/enumerate.go b/pkg/runner/enumerate.go similarity index 82% rename from v2/pkg/runner/enumerate.go rename to pkg/runner/enumerate.go index f71a347dd..bd8953e56 100644 --- a/v2/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -67,6 +67,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error) case subscraping.Subdomain: subdomain := replacer.Replace(result.Value) + // check if this subdomain is actually a wildcard subdomain + // that may have furthur subdomains associated with it + isWildcard := strings.Contains(result.Value, "*."+subdomain) // Validate the subdomain found and remove wildcards from if !strings.HasSuffix(subdomain, "."+domain) { @@ -90,10 +93,20 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // send the subdomain for resolution. if _, ok := uniqueMap[subdomain]; ok { skippedCounts[result.Source]++ + // even if it is duplicate if it was not marked as wildcard before but this source says it is wildcard + // then we should mark it as wildcard + if !uniqueMap[subdomain].WildcardCertificate && isWildcard { + val := uniqueMap[subdomain] + val.WildcardCertificate = true + uniqueMap[subdomain] = val + } continue } - hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source} + hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source, WildcardCertificate: isWildcard} + if r.options.ResultCallback != nil && !r.options.RemoveWildcard { + r.options.ResultCallback(&hostEntry) + } uniqueMap[subdomain] = hostEntry // If the user asked to remove wildcard then send on the resolve @@ -109,6 +122,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string if r.options.RemoveWildcard { close(resolutionPool.Tasks) } + wg.Done() }() @@ -125,9 +139,22 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // Add the found subdomain to a map. if _, ok := foundResults[result.Host]; !ok { foundResults[result.Host] = result + if r.options.ResultCallback != nil { + r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate}) + } } } } + + // Merge wildcard certificate information from uniqueMap into foundResults + // This handles cases where a later source marked a subdomain as wildcard + // after it was already sent to the resolution pool + for host, result := range foundResults { + if entry, ok := uniqueMap[host]; ok && entry.WildcardCertificate && !result.WildcardCertificate { + result.WildcardCertificate = true + foundResults[host] = result + } + } } wg.Wait() outputWriter := NewOutputWriter(r.options.JSON) @@ -162,17 +189,6 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string numberOfSubDomains = len(uniqueMap) } - if r.options.ResultCallback != nil { - if r.options.RemoveWildcard { - for host, result := range foundResults { - r.options.ResultCallback(&resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source}) - } - } else { - for _, v := range uniqueMap { - r.options.ResultCallback(&v) - } - } - } gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration) if r.options.Statistics { diff --git a/v2/pkg/runner/enumerate_test.go b/pkg/runner/enumerate_test.go similarity index 100% rename from v2/pkg/runner/enumerate_test.go rename to pkg/runner/enumerate_test.go diff --git a/v2/pkg/runner/initialize.go b/pkg/runner/initialize.go similarity index 100% rename from v2/pkg/runner/initialize.go rename to pkg/runner/initialize.go diff --git a/v2/pkg/runner/options.go b/pkg/runner/options.go similarity index 93% rename from v2/pkg/runner/options.go rename to pkg/runner/options.go index 5e3d4b172..946c3027b 100644 --- a/v2/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -15,6 +15,8 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + envutil "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" @@ -23,8 +25,8 @@ import ( var ( configDir = folderutil.AppConfigDirOrDefault(".", "subfinder") - defaultConfigLocation = filepath.Join(configDir, "config.yaml") - defaultProviderConfigLocation = filepath.Join(configDir, "provider-config.yaml") + defaultConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_CONFIG", filepath.Join(configDir, "config.yaml")) + defaultProviderConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_PROVIDER_CONFIG", filepath.Join(configDir, "provider-config.yaml")) ) // Options contains the configuration options for tuning @@ -222,16 +224,20 @@ func (options *Options) loadProvidersFrom(location string) { func listSources(options *Options) { gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) - gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n") + gologger.Info().Msgf("Sources marked with an * require key(s) or token(s) to work.\n") + gologger.Info().Msgf("Sources marked with a ~ optionally support key(s) for better results.\n") gologger.Info().Msgf("You can modify %s to configure your keys/tokens.\n\n", options.ProviderConfig) for _, source := range passive.AllSources { - message := "%s\n" sourceName := source.Name() - if source.NeedsKey() { - message = "%s *\n" + switch source.KeyRequirement() { + case subscraping.RequiredKey: + gologger.Silent().Msgf("%s *\n", sourceName) + case subscraping.OptionalKey: + gologger.Silent().Msgf("%s ~\n", sourceName) + default: + gologger.Silent().Msgf("%s\n", sourceName) } - gologger.Silent().Msgf(message, sourceName) } } @@ -244,6 +250,7 @@ func (options *Options) preProcessDomains() { var defaultRateLimits = []string{ "github=30/m", "fullhunt=60/m", + "pugrecon=10/s", fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", "shodan=1/s", @@ -258,4 +265,5 @@ var defaultRateLimits = []string{ // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", + "urlscan=1/s", } diff --git a/v2/pkg/runner/outputter.go b/pkg/runner/outputter.go similarity index 79% rename from v2/pkg/runner/outputter.go rename to pkg/runner/outputter.go index faef0cae0..2a41f5127 100644 --- a/v2/pkg/runner/outputter.go +++ b/pkg/runner/outputter.go @@ -19,22 +19,25 @@ type OutputWriter struct { } type jsonSourceResult struct { - Host string `json:"host"` - Input string `json:"input"` - Source string `json:"source"` + Host string `json:"host"` + Input string `json:"input"` + Source string `json:"source"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourceIPResult struct { - Host string `json:"host"` - IP string `json:"ip"` - Input string `json:"input"` - Source string `json:"source"` + Host string `json:"host"` + IP string `json:"ip"` + Input string `json:"input"` + Source string `json:"source"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourcesResult struct { - Host string `json:"host"` - Input string `json:"input"` - Sources []string `json:"sources"` + Host string `json:"host"` + Input string `json:"input"` + Sources []string `json:"sources"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } // NewOutputWriter creates a new OutputWriter @@ -97,7 +100,9 @@ func writePlainHostIP(_ string, results map[string]resolve.Result, writer io.Wri _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() @@ -115,7 +120,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io. data.IP = result.IP data.Input = input data.Source = result.Source - + data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(&data) if err != nil { return err @@ -128,7 +133,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io. func (o *OutputWriter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error { hosts := make(map[string]resolve.HostEntry) for host, result := range results { - hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source} + hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate} } return o.WriteHost(input, hosts, writer) @@ -155,7 +160,9 @@ func writePlainHost(_ string, results map[string]resolve.HostEntry, writer io.Wr _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() @@ -171,6 +178,7 @@ func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io data.Host = result.Host data.Input = input data.Source = result.Source + data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(data) if err != nil { return err @@ -219,16 +227,19 @@ func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, wr for host, sources := range sourceMap { sb.WriteString(host) sb.WriteString(",[") - sourcesString := "" + var sourcesString strings.Builder for source := range sources { - sourcesString += source + "," + sourcesString.WriteString(source) + sourcesString.WriteRune(',') } - sb.WriteString(strings.Trim(sourcesString, ", ")) + sb.WriteString(strings.TrimSuffix(sourcesString.String(), ",")) sb.WriteString("]\n") _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() diff --git a/v2/pkg/runner/runner.go b/pkg/runner/runner.go similarity index 89% rename from v2/pkg/runner/runner.go rename to pkg/runner/runner.go index ce9350419..b9dd844d3 100644 --- a/v2/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -10,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/projectdiscovery/gologger" contextutil "github.com/projectdiscovery/utils/context" @@ -61,11 +62,17 @@ func NewRunner(options *Options) (*Runner, error) { Custom: mapsutil.SyncLockMap[string, uint]{ Map: make(map[string]uint), }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, } for source, sourceRateLimit := range options.RateLimits.AsMap() { if sourceRateLimit.MaxCount > 0 && sourceRateLimit.MaxCount <= math.MaxUint { _ = runner.rateLimit.Custom.Set(source, sourceRateLimit.MaxCount) + if sourceRateLimit.Duration > 0 { + _ = runner.rateLimit.CustomDuration.Set(source, sourceRateLimit.Duration) + } } } @@ -94,7 +101,9 @@ func (r *Runner) RunEnumerationWithCtx(ctx context.Context) error { return err } err = r.EnumerateMultipleDomainsWithCtx(ctx, f, outputs) - f.Close() + if closeErr := f.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", r.options.DomainsFile, closeErr) + } return err } @@ -139,7 +148,9 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) - file.Close() + if closeErr := file.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", r.options.OutputFile, closeErr) + } } else if r.options.OutputDirectory != "" { outputFile := path.Join(r.options.OutputDirectory, domain) if r.options.JSON { @@ -157,7 +168,9 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) - file.Close() + if closeErr := file.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", outputFile, closeErr) + } } else { _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers) } diff --git a/v2/pkg/runner/stats.go b/pkg/runner/stats.go similarity index 79% rename from v2/pkg/runner/stats.go rename to pkg/runner/stats.go index d34b20f82..ccaa13e32 100644 --- a/v2/pkg/runner/stats.go +++ b/pkg/runner/stats.go @@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) { if sourceStats.Skipped { skipped = append(skipped, fmt.Sprintf(" %s", source)) } else { - lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors)) + lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors)) } } if len(lines) > 0 { - gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56)) + gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68)) gologger.Print().Msg(strings.Join(lines, "\n")) gologger.Print().Msgf("\n") } diff --git a/v2/pkg/runner/util.go b/pkg/runner/util.go similarity index 100% rename from v2/pkg/runner/util.go rename to pkg/runner/util.go diff --git a/v2/pkg/runner/validate.go b/pkg/runner/validate.go similarity index 100% rename from v2/pkg/runner/validate.go rename to pkg/runner/validate.go diff --git a/v2/pkg/subscraping/agent.go b/pkg/subscraping/agent.go similarity index 95% rename from v2/pkg/subscraping/agent.go rename to pkg/subscraping/agent.go index 0e2b7f13a..6217ce58d 100644 --- a/v2/pkg/subscraping/agent.go +++ b/pkg/subscraping/agent.go @@ -46,7 +46,7 @@ func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLi Timeout: time.Duration(timeout) * time.Second, } - session := &Session{Client: client} + session := &Session{Client: client, Timeout: timeout} // Initiate rate limit instance session.MultiRateLimiter = multiRateLimiter @@ -119,7 +119,9 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) { gologger.Warning().Msgf("Could not discard response body: %s\n", err) return } - response.Body.Close() + if closeErr := response.Body.Close(); closeErr != nil { + gologger.Warning().Msgf("Could not close response body: %s\n", closeErr) + } } } diff --git a/v2/pkg/subscraping/doc.go b/pkg/subscraping/doc.go similarity index 100% rename from v2/pkg/subscraping/doc.go rename to pkg/subscraping/doc.go diff --git a/v2/pkg/subscraping/extractor.go b/pkg/subscraping/extractor.go similarity index 100% rename from v2/pkg/subscraping/extractor.go rename to pkg/subscraping/extractor.go diff --git a/v2/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go similarity index 68% rename from v2/pkg/subscraping/sources/alienvault/alienvault.go rename to pkg/subscraping/sources/alienvault/alienvault.go index 857b2c78a..c383e7354 100644 --- a/v2/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -23,6 +23,9 @@ type Source struct { timeTaken time.Duration results int errors int + requests int + apiKeys []string + skipped bool } // Run function returns all subdomains found with the service @@ -30,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -37,7 +41,15 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + s.requests++ + resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "", + map[string]string{"Authorization": "Bearer " + randomApiKey}) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -51,10 +63,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Error != "" { results <- subscraping.Result{ @@ -64,8 +76,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, record := range response.PassiveDNS { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: + s.results++ + } } }() @@ -85,18 +101,24 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, + Skipped: s.skipped, } } diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go similarity index 75% rename from v2/pkg/subscraping/sources/anubis/anubis.go rename to pkg/subscraping/sources/anubis/anubis.go index 31e31a4c2..fa951457d 100644 --- a/v2/pkg/subscraping/sources/anubis/anubis.go +++ b/pkg/subscraping/sources/anubis/anubis.go @@ -4,6 +4,7 @@ package anubis import ( "context" "fmt" + "net/http" "time" jsoniter "github.com/json-iterator/go" @@ -16,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -23,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -30,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -38,20 +42,29 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + if resp.StatusCode != http.StatusOK { + session.DiscardHTTPResponse(resp) + return + } + var subdomains []string err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: + s.results++ + } } }() @@ -72,8 +85,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -84,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/v2/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go similarity index 81% rename from v2/pkg/subscraping/sources/bevigil/bevigil.go rename to pkg/subscraping/sources/bevigil/bevigil.go index 893db4913..22197c7ff 100644 --- a/v2/pkg/subscraping/sources/bevigil/bevigil.go +++ b/pkg/subscraping/sources/bevigil/bevigil.go @@ -21,6 +21,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) + s.requests++ resp, err := session.Get(ctx, getUrl, "", map[string]string{ "X-Access-Token": randomApiKey, "User-Agent": "subfinder", }) @@ -59,19 +62,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response.Subdomains) > 0 { subdomains = response.Subdomains } for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() @@ -90,8 +97,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -102,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go similarity index 86% rename from v2/pkg/subscraping/sources/bufferover/bufferover.go rename to pkg/subscraping/sources/bufferover/bufferover.go index bbfc884a3..3c726607f 100644 --- a/v2/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { + s.requests++ resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) if err != nil && resp == nil { @@ -69,11 +72,11 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) metaErrors := bufforesponse.Meta.Errors @@ -96,8 +99,12 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s for _, subdomain := range subdomains { for _, value := range session.Extractor.Extract(subdomain) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -115,8 +122,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -127,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go similarity index 81% rename from v2/pkg/subscraping/sources/builtwith/builtwith.go rename to pkg/subscraping/sources/builtwith/builtwith.go index 24f2e60cc..c500e6dc1 100644 --- a/v2/pkg/subscraping/sources/builtwith/builtwith.go +++ b/pkg/subscraping/sources/builtwith/builtwith.go @@ -35,6 +35,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -68,14 +71,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, result := range data.Results { for _, path := range result.Result.Paths { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}: + s.results++ + } } } }() @@ -96,8 +103,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -108,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go similarity index 82% rename from v2/pkg/subscraping/sources/c99/c99.go rename to pkg/subscraping/sources/c99/c99.go index bfd7fcf55..54f8534c1 100644 --- a/v2/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,13 +52,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) @@ -76,8 +84,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, data := range response.Subdomains { if !strings.HasPrefix(data.Subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}: + s.results++ + } } } }() @@ -98,8 +110,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -110,6 +126,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go new file mode 100644 index 000000000..e913c8645 --- /dev/null +++ b/pkg/subscraping/sources/censys/censys.go @@ -0,0 +1,234 @@ +// Package censys logic +package censys + +import ( + "bytes" + "context" + "net/http" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // maxCensysPages is the maximum number of pages to fetch from the API + maxCensysPages = 10 + // maxPerPage is the maximum number of results per page + maxPerPage = 100 + // baseURL is the Censys Platform API base URL + baseURL = "https://api.platform.censys.io" + // searchEndpoint is the global data search query endpoint + searchEndpoint = "/v3/global/search/query" + // queryPrefix is the Censys query language prefix for certificate name search + queryPrefix = "cert.names: " + // authHeaderPrefix is the Bearer token prefix for Authorization header + authHeaderPrefix = "Bearer " + // contentTypeJSON is the Content-Type header value for JSON + contentTypeJSON = "application/json" + // orgIDHeader is the header name for organization ID + orgIDHeader = "X-Organization-ID" +) + +// apiKey holds the Personal Access Token and optional Organization ID +type apiKey struct { + pat string + orgID string +} + +// Platform API request body +type searchRequest struct { + Query string `json:"query"` + Fields []string `json:"fields,omitempty"` + PageSize int `json:"page_size,omitempty"` + Cursor string `json:"cursor,omitempty"` +} + +// Platform API response structures +type response struct { + Result result `json:"result"` +} + +type result struct { + Hits []hit `json:"hits"` + TotalHits int64 `json:"total_hits"` + NextPageToken string `json:"next_page_token"` +} + +type hit struct { + CertificateV1 certificateV1 `json:"certificate_v1"` +} + +type certificateV1 struct { + Resource resource `json:"resource"` +} + +type resource struct { + Names []string `json:"names"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + // PickRandom selects a random API key from configured keys. + // This enables load balancing when users configure multiple PATs + // (e.g., CENSYS_API_KEY=pat1:org1,pat2:org2) to distribute requests + // and avoid hitting rate limits on a single key. + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.pat == "" { + s.skipped = true + return + } + + apiURL := baseURL + searchEndpoint + cursor := "" + currentPage := 1 + + for { + select { + case <-ctx.Done(): + return + default: + } + + reqBody := searchRequest{ + Query: queryPrefix + domain, + Fields: []string{"cert.names"}, + PageSize: maxPerPage, + } + if cursor != "" { + reqBody.Cursor = cursor + } + + bodyBytes, err := jsoniter.Marshal(reqBody) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + headers := map[string]string{ + "Content-Type": contentTypeJSON, + "Authorization": authHeaderPrefix + randomApiKey.pat, + } + // Add Organization ID header if provided + if randomApiKey.orgID != "" { + headers[orgIDHeader] = randomApiKey.orgID + } + + s.requests++ + resp, err := session.HTTPRequest( + ctx, + http.MethodPost, + apiURL, + "", + headers, + bytes.NewReader(bodyBytes), + subscraping.BasicAuth{}, + ) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var censysResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) + _ = resp.Body.Close() + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, hit := range censysResponse.Result.Hits { + for _, name := range hit.CertificateV1.Resource.Names { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}: + s.results++ + } + } + } + + cursor = censysResponse.Result.NextPageToken + if cursor == "" || currentPage >= maxCensysPages { + break + } + currentPage++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "censys" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +// AddApiKeys parses and adds API keys. +// Format: "PAT:ORG_ID" where ORG_ID is required for paid accounts. +// Example: "censys_xxx_token:12345678-91011-1213" +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(pat, orgID string) apiKey { + return apiKey{pat: pat, orgID: orgID} + }) + // Also support single PAT without org ID for free users + for _, key := range keys { + if !strings.Contains(key, ":") && key != "" { + s.apiKeys = append(s.apiKeys, apiKey{pat: key, orgID: ""}) + } + } +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/pkg/subscraping/sources/censys/censys_test.go b/pkg/subscraping/sources/censys/censys_test.go new file mode 100644 index 000000000..f928698f0 --- /dev/null +++ b/pkg/subscraping/sources/censys/censys_test.go @@ -0,0 +1,137 @@ +package censys + +import ( + "context" + "math" + "net/http" + "testing" + "time" + + "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createTestMultiRateLimiter creates a MultiLimiter for testing +func createTestMultiRateLimiter(ctx context.Context) *ratelimit.MultiLimiter { + mrl, _ := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ + Key: "censys", + IsUnlimited: false, + MaxCount: math.MaxInt32, + Duration: time.Millisecond, + }) + return mrl +} + +func TestCensysSource_NoApiKey(t *testing.T) { + source := &Source{} + // Don't add any API keys + + ctx := context.Background() + multiRateLimiter := createTestMultiRateLimiter(ctx) + session := &subscraping.Session{ + Client: http.DefaultClient, + MultiRateLimiter: multiRateLimiter, + } + + ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, "censys") + results := source.Run(ctxWithValue, "example.com", session) + + // Collect all results + var resultCount int + for range results { + resultCount++ + } + + // Should be skipped when no API key + stats := source.Statistics() + assert.True(t, stats.Skipped, "expected source to be skipped without API key") + assert.Equal(t, 0, resultCount, "expected no results when skipped") +} + +func TestCensysSource_ContextCancellation(t *testing.T) { + source := &Source{} + // Add a key with PAT:ORG_ID format + source.AddApiKeys([]string{"test_pat:test_org_id"}) + + ctx := context.Background() + multiRateLimiter := createTestMultiRateLimiter(ctx) + session := &subscraping.Session{ + Client: http.DefaultClient, + MultiRateLimiter: multiRateLimiter, + } + + // Create a context that will be cancelled + ctxCancellable, cancel := context.WithCancel(ctx) + ctxWithValue := context.WithValue(ctxCancellable, subscraping.CtxSourceArg, "censys") + + results := source.Run(ctxWithValue, "example.com", session) + + // Cancel immediately + cancel() + + // Should exit quickly without blocking + done := make(chan struct{}) + go func() { + for range results { + // drain + } + close(done) + }() + + select { + case <-done: + // Good - completed quickly + case <-time.After(2 * time.Second): + t.Fatal("context cancellation did not stop the source in time") + } +} + +func TestCensysSource_Metadata(t *testing.T) { + source := &Source{} + + assert.Equal(t, "censys", source.Name()) + assert.True(t, source.IsDefault()) + assert.False(t, source.HasRecursiveSupport()) + assert.True(t, source.NeedsKey()) +} + +func TestCensysSource_AddApiKeys(t *testing.T) { + t.Run("PAT with OrgID", func(t *testing.T) { + source := &Source{} + keys := []string{"pat_token_1:org_id_1", "pat_token_2:org_id_2"} + source.AddApiKeys(keys) + + require.Len(t, source.apiKeys, 2) + assert.Equal(t, "pat_token_1", source.apiKeys[0].pat) + assert.Equal(t, "org_id_1", source.apiKeys[0].orgID) + assert.Equal(t, "pat_token_2", source.apiKeys[1].pat) + assert.Equal(t, "org_id_2", source.apiKeys[1].orgID) + }) + + t.Run("PAT without OrgID (free user)", func(t *testing.T) { + source := &Source{} + keys := []string{"pat_token_only"} + source.AddApiKeys(keys) + + require.Len(t, source.apiKeys, 1) + assert.Equal(t, "pat_token_only", source.apiKeys[0].pat) + assert.Equal(t, "", source.apiKeys[0].orgID) + }) +} + +func TestCensysSource_Statistics(t *testing.T) { + source := &Source{ + errors: 2, + results: 10, + timeTaken: 5 * time.Second, + skipped: false, + } + + stats := source.Statistics() + assert.Equal(t, 2, stats.Errors) + assert.Equal(t, 10, stats.Results) + assert.Equal(t, 5*time.Second, stats.TimeTaken) + assert.False(t, stats.Skipped) +} diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go similarity index 79% rename from v2/pkg/subscraping/sources/certspotter/certspotter.go rename to pkg/subscraping/sources/certspotter/certspotter.go index cf49d1dc9..86e0160e9 100644 --- a/v2/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Authorization": "Bearer " + randomApiKey} cookies := "" + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -59,27 +62,36 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, cert := range response { for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } - // if the number of responses is zero, close the channel and return. if len(response) == 0 { return } id := response[len(response)-1].ID for { + select { + case <-ctx.Done(): + return + default: + } reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) + s.requests++ resp, err := session.Get(ctx, reqURL, cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -92,10 +104,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response) == 0 { break @@ -103,8 +115,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, cert := range response { for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } @@ -128,8 +144,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -140,6 +160,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go similarity index 81% rename from v2/pkg/subscraping/sources/chaos/chaos.go rename to pkg/subscraping/sources/chaos/chaos.go index 2cc873679..13824d91c 100644 --- a/v2/pkg/subscraping/sources/chaos/chaos.go +++ b/pkg/subscraping/sources/chaos/chaos.go @@ -16,14 +16,16 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } // Run function returns all subdomains found with the service -func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { +func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,9 +40,15 @@ func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) < } chaosClient := chaos.New(randomApiKey) + s.requests++ for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ Domain: domain, }) { + select { + case <-ctx.Done(): + return + default: + } if result.Error != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} s.errors++ @@ -69,8 +77,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -81,6 +93,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go similarity index 81% rename from v2/pkg/subscraping/sources/chinaz/chinaz.go rename to pkg/subscraping/sources/chinaz/chinaz.go index d7063973f..7e2045aac 100644 --- a/v2/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -48,16 +51,25 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := io.ReadAll(resp.Body) - resp.Body.Close() + session.DiscardHTTPResponse(resp) SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") if SubdomainList.ToBool() { _data := []byte(SubdomainList.ToString()) for i := 0; i < SubdomainList.Size(); i++ { + select { + case <-ctx.Done(): + return + default: + } subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } else { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -82,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -94,6 +110,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go similarity index 82% rename from v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go rename to pkg/subscraping/sources/commoncrawl/commoncrawl.go index 34e50be9e..9dd4d4256 100644 --- a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, indexURL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -59,13 +62,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) years := make([]string, 0) - for i := 0; i < maxYearsBack; i++ { + for i := range maxYearsBack { years = append(years, strconv.Itoa(year-i)) } @@ -105,8 +108,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -117,6 +124,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } @@ -128,6 +136,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se return false default: var headers = map[string]string{"Host": "index.commoncrawl.org"} + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -138,6 +147,12 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return false + default: + } line := scanner.Text() if line == "" { continue @@ -145,17 +160,21 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { if subdomain != "" { - // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return false + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) return true } } diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go similarity index 75% rename from v2/pkg/subscraping/sources/crtsh/crtsh.go rename to pkg/subscraping/sources/crtsh/crtsh.go index 4d008349f..a22b5bdc7 100644 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -14,6 +14,7 @@ import ( // postgres driver _ "github.com/lib/pq" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" contextutil "github.com/projectdiscovery/utils/context" ) @@ -28,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -35,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -53,14 +56,21 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int { - db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") + // connect_timeout: limits connection establishment time (in seconds) + // statement_timeout: limits query execution time (in milliseconds) + connStr := fmt.Sprintf("host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes connect_timeout=%d statement_timeout=%d", session.Timeout, session.Timeout*1000) + db, err := sql.Open("postgres", connStr) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return 0 } - defer db.Close() + defer func() { + if closeErr := db.Close(); closeErr != nil { + gologger.Warning().Msgf("Could not close database connection: %s\n", closeErr) + } + }() limitClause := "" if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok { @@ -109,8 +119,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio var count int var data string - // Parse all the rows getting subdomains for rows.Next() { + select { + case <-ctx.Done(): + return count + default: + } err := rows.Scan(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -119,11 +133,15 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } count++ - for _, subdomain := range strings.Split(data, "\n") { + for subdomain := range strings.SplitSeq(data, "\n") { for _, value := range session.Extractor.Extract(subdomain) { if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return count + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -132,6 +150,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -145,18 +164,27 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return false } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, subdomain := range subdomains { - for _, sub := range strings.Split(subdomain.NameValue, "\n") { + select { + case <-ctx.Done(): + return true + default: + } + for sub := range strings.SplitSeq(subdomain.NameValue, "\n") { for _, value := range session.Extractor.Extract(sub) { if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return true + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -178,8 +206,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -190,6 +222,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go new file mode 100644 index 000000000..5beae8211 --- /dev/null +++ b/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -0,0 +1,145 @@ +package digitalyama + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +type digitalYamaResponse struct { + Query string `json:"query"` + Count int `json:"count"` + Subdomains []string `json:"subdomains"` + UsageSummary struct { + QueryCost float64 `json:"query_cost"` + CreditsRemaining float64 `json:"credits_remaining"` + } `json:"usage_summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) + s.requests++ + resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + if resp.StatusCode != 200 { + var errResponse struct { + Detail []struct { + Loc []string `json:"loc"` + Msg string `json:"msg"` + Type string `json:"type"` + } `json:"detail"` + } + err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + s.errors++ + return + } + if len(errResponse.Detail) > 0 { + errMsg := errResponse.Detail[0].Msg + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)} + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + } + s.errors++ + return + } + + var response digitalYamaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range response.Subdomains { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "digitalyama" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go similarity index 74% rename from v2/pkg/subscraping/sources/digitorus/digitorus.go rename to pkg/subscraping/sources/digitorus/digitorus.go index 6cf61dbcf..226469a2f 100644 --- a/v2/pkg/subscraping/sources/digitorus/digitorus.go +++ b/pkg/subscraping/sources/digitorus/digitorus.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -32,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound { @@ -41,20 +44,32 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } subdomains := session.Extractor.Extract(line) for _, subdomain := range subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."), + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, ".")}: + s.results++ } - s.results++ } } }() @@ -75,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -87,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go similarity index 85% rename from v2/pkg/subscraping/sources/dnsdb/dnsdb.go rename to pkg/subscraping/sources/dnsdb/dnsdb.go index 2b18b8568..ce299c1c7 100644 --- a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results uint64 + requests int skipped bool } @@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -70,6 +72,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "Accept": "application/x-ndjson", } + s.requests++ offsetMax, err := getMaxOffset(ctx, session, headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -85,8 +88,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se queryParams.Add("swclient", "subfinder") for { + select { + case <-ctx.Done(): + return + default: + } url := urlTemplate + queryParams.Encode() + s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -98,13 +107,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var respCond string reader := bufio.NewReader(resp.Body) for { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } n, err := reader.ReadBytes('\n') if err == io.EOF { break } else if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -113,25 +128,22 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - // Condition is a scalar enum of string values: {β€œbegin”, β€œongoing”, β€œsucceeded”, β€œlimited”, β€œfailed”}. - // "begin" will be the initiating Condition, this can be safely ignored. The data of interest will be in - // objects with Condition "" or "ongoing". Conditions "succeeded", "limited", and "failed" are terminating conditions. - // See https://www.domaintools.com/resources/user-guides/farsight-streaming-api-framing-protocol-documentation/ - // for more details respCond = response.Condition if respCond == "" || respCond == "ongoing" { if response.Obj.Name != "" { - results <- subscraping.Result{ - Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, "."), + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, ".")}: + s.results++ } - s.results++ } } else if respCond != "begin" { - // if the respCond is not "", "ongoing", or "begin", then it is a terminating condition, so break out of the loop break } } @@ -154,7 +166,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ } - resp.Body.Close() + session.DiscardHTTPResponse(resp) break } }() @@ -175,8 +187,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -187,6 +203,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: int(s.results), + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go similarity index 81% rename from v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go rename to pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 2155e31cf..fec6f73a3 100644 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -25,6 +25,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -53,20 +56,24 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response response err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } for _, record := range append(response.A, response.Ns...) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}: + s.results++ + } } }() @@ -87,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -99,6 +110,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go similarity index 70% rename from v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go rename to pkg/subscraping/sources/dnsrepo/dnsrepo.go index 903f34985..14e1084c6 100644 --- a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -17,17 +17,19 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } type DnsRepoResponse []struct { - Domain string + Domain string `json:"domain"` } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,7 +42,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.skipped = true return } - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", randomApiKey, domain)) + + randomApiInfo := strings.Split(randomApiKey, ":") + if len(randomApiInfo) != 2 { + s.skipped = true + return + } + + token := randomApiInfo[0] + apiKey := randomApiInfo[1] + + s.requests++ + resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -54,7 +67,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) var result DnsRepoResponse err = json.Unmarshal(responseData, &result) if err != nil { @@ -64,10 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, sub := range result { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, "."), + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")}: + s.results++ } - s.results++ } }() @@ -88,8 +103,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -100,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/domainsproject/domainsproject.go similarity index 56% rename from v2/pkg/subscraping/sources/quake/quake.go rename to pkg/subscraping/sources/domainsproject/domainsproject.go index 075d35e13..b6c7eab8f 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -1,8 +1,7 @@ -// Package quake logic -package quake +// Package domainsproject logic +package domainsproject import ( - "bytes" "context" "fmt" "strings" @@ -13,37 +12,32 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -type quakeResults struct { - Code int `json:"code"` - Message string `json:"message"` - Data []struct { - Service struct { - HTTP struct { - Host string `json:"host"` - } `json:"http"` - } - } `json:"data"` - Meta struct { - Pagination struct { - Total int `json:"total"` - } `json:"pagination"` - } `json:"meta"` -} - // Source is the passive scraping agent type Source struct { - apiKeys []string + apiKeys []apiKey timeTaken time.Duration errors int results int + requests int skipped bool } +type apiKey struct { + username string + password string +} + +type domainsProjectResponse struct { + Domains []string `json:"domains"` + Error string `json:"error"` +} + // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -52,16 +46,22 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { + if randomApiKey.username == "" || randomApiKey.password == "" { s.skipped = true return } - // quake api doc https://quake.360.cn/quake/#/help - var requestBody = []byte(fmt.Sprintf(`{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)) - resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ - "Content-Type": "application/json", "X-QuakeToken": randomApiKey, - }, bytes.NewReader(requestBody)) + searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain) + s.requests++ + resp, err := session.HTTPRequest( + ctx, + "GET", + searchURL, + "", + nil, + nil, + subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, + ) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -69,32 +69,37 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - var response quakeResults + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + var response domainsProjectResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() return } - resp.Body.Close() - if response.Code != 0 { + if response.Error != "" { results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), } s.errors++ return } - if response.Meta.Pagination.Total > 0 { - for _, quakeDomain := range response.Data { - subdomain := quakeDomain.Service.HTTP.Host - if strings.ContainsAny(subdomain, "ζš‚ζ— ζƒι™") { - subdomain = "" + for _, subdomain := range response.Domains { + if !strings.HasPrefix(subdomain, ".") { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ } } }() @@ -104,7 +109,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Name returns the name of the source func (s *Source) Name() string { - return "quake" + return "domainsproject" } func (s *Source) IsDefault() bool { @@ -115,18 +120,25 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go new file mode 100644 index 000000000..194952b84 --- /dev/null +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -0,0 +1,211 @@ +package driftnet + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the base URL for the driftnet API + baseURL = "https://api.driftnet.io/v1/" + + // summaryLimit is the size of the summary limit that we send to the API + summaryLimit = 10000 +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 + skipped bool +} + +// endpointConfig describes a driftnet endpoint that can used +type endpointConfig struct { + // The API endpoint to be touched + endpoint string + + // The API parameter used for query + param string + + // The context that we should restrict to in results from this endpoint + context string +} + +// endpoints is a set of endpoint configs +var endpoints = []endpointConfig{ + {"ct/log", "field=host:", "cert-dns-name"}, + {"scan/protocols", "field=host:", "cert-dns-name"}, + {"scan/domains", "field=host:", "cert-dns-name"}, + {"domain/rdns", "host=", "dns-ptr"}, +} + +// summaryResponse is an API response +type summaryResponse struct { + Summary struct { + Other int `json:"other"` + Values map[string]int `json:"values"` + } `json:"summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + // Final results channel + results := make(chan subscraping.Result) + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) + + // Waitgroup for subsources + var wg sync.WaitGroup + wg.Add(len(endpoints)) + + // Map for dedupe between subsources + dedupe := sync.Map{} + + // Close down results when all subsources finished + go func(startTime time.Time) { + wg.Wait() + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + // Start up requests for all subsources + for i := range endpoints { + go s.runSubsource(ctx, domain, session, results, &wg, &dedupe, endpoints[i]) + } + + // Return the results channel + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "driftnet" +} + +// IsDefault indicates that this source should used as part of the default execution. +func (s *Source) IsDefault() bool { + return true +} + +// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains +func (s *Source) HasRecursiveSupport() bool { + return true +} + +// KeyRequirement indicates that we need an API key +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +// NeedsKey indicates that we need an API key +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +// AddApiKeys provides us with the API key(s) +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns statistics about the scraping process +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} + +// runSubsource queries a specific driftnet endpoint for subdomains and sends results to the channel +func (s *Source) runSubsource(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result, wg *sync.WaitGroup, dedupe *sync.Map, epConfig endpointConfig) { + // Default headers + headers := map[string]string{ + "accept": "application/json", + } + + // Pick an API key + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["authorization"] = "Bearer " + randomApiKey + } + + // Request + requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) + s.requests.Add(1) + resp, err := session.Get(ctx, requestURL, "", headers) + if err != nil { + // HTTP 204 is not an error from the Driftnet API + if resp == nil || resp.StatusCode != http.StatusNoContent { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors.Add(1) + } + + wg.Done() + return + } + + defer session.DiscardHTTPResponse(resp) + + // 204 means no results, any other response code is an error + if resp.StatusCode != 200 { + if resp.StatusCode != 204 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} + s.errors.Add(1) + } + + wg.Done() + return + } + + // Parse and return results + var summary summaryResponse + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&summary) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors.Add(1) + wg.Done() + return + } + + for subdomain := range summary.Summary.Values { + select { + case <-ctx.Done(): + wg.Done() + return + default: + } + if !strings.HasSuffix(subdomain, "."+domain) { + continue + } + + if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { + select { + case <-ctx.Done(): + wg.Done() + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) + } + } + } + + // Complete! + wg.Done() +} diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go similarity index 87% rename from v2/pkg/subscraping/sources/fofa/fofa.go rename to pkg/subscraping/sources/fofa/fofa.go index 9a5294c75..48ad360b7 100644 --- a/v2/pkg/subscraping/sources/fofa/fofa.go +++ b/pkg/subscraping/sources/fofa/fofa.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -54,7 +56,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // fofa api doc https://fofa.info/static_pages/api_help - qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -68,10 +71,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Error { results <- subscraping.Result{ @@ -83,6 +86,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if response.Size > 0 { for _, subdomain := range response.Results { + select { + case <-ctx.Done(): + return + default: + } if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { subdomain = subdomain[strings.Index(subdomain, "//")+2:] } @@ -112,8 +120,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -126,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go similarity index 81% rename from v2/pkg/subscraping/sources/fullhunt/fullhunt.go rename to pkg/subscraping/sources/fullhunt/fullhunt.go index 10c054e89..237ae2861 100644 --- a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -56,13 +59,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range response.Hosts { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: + s.results++ + } } }() @@ -82,8 +89,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -94,6 +105,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go similarity index 80% rename from v2/pkg/subscraping/sources/github/github.go rename to pkg/subscraping/sources/github/github.go index 6034b2dbc..e92d9fdf9 100644 --- a/v2/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -41,16 +42,18 @@ type response struct { type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) go func() { defer func(startTime time.Time) { @@ -86,11 +89,12 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } // Initial request to GitHub search + s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil && !isForbidden { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -100,7 +104,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * if isForbidden && ratelimitRemaining == 0 { retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) tokens.setCurrentTokenExceeded(retryAfterSeconds) - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) } @@ -111,17 +115,17 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() + s.errors.Add(1) + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -129,11 +133,16 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * linksHeader := linkheader.Parse(resp.Header.Get("Link")) // Process the next link recursively for _, link := range linksHeader { + select { + case <-ctx.Done(): + return + default: + } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) @@ -147,11 +156,22 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r errChan := make(chan error, len(items)) for _, responseItem := range items { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } wg.Add(1) go func(responseItem item) { defer wg.Done() - // find subdomains in code + select { + case <-ctx.Done(): + return + default: + } + + s.requests.Add(1) resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { @@ -164,23 +184,42 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r if resp.StatusCode == http.StatusOK { scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } line := scanner.Text() if line == "" { continue } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) + } } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) } - // find subdomains in text matches for _, textMatch := range responseItem.TextMatches { + select { + case <-ctx.Done(): + return + default: + } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) + } } } }(responseItem) @@ -231,8 +270,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -241,8 +284,9 @@ func (s *Source) AddApiKeys(keys []string) { func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/github/tokenmanager.go b/pkg/subscraping/sources/github/tokenmanager.go similarity index 100% rename from v2/pkg/subscraping/sources/github/tokenmanager.go rename to pkg/subscraping/sources/github/tokenmanager.go diff --git a/v2/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go similarity index 85% rename from v2/pkg/subscraping/sources/gitlab/gitlab.go rename to pkg/subscraping/sources/gitlab/gitlab.go index 9477b0bdf..944f4005a 100644 --- a/v2/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -20,8 +21,9 @@ import ( type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 skipped bool } @@ -35,8 +37,9 @@ type item struct { // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) go func() { defer func(startTime time.Time) { @@ -66,21 +69,22 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * default: } + s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var items []item err = jsoniter.NewDecoder(resp.Body).Decode(&items) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -91,13 +95,14 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * go func(item item) { // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) + s.requests.Add(1) resp, err := session.Get(ctx, fileUrl, "", headers) if err != nil { if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { session.DiscardHTTPResponse(resp) results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } } @@ -111,24 +116,27 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } for _, subdomain := range domainRegexp.FindAllString(line, -1) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + s.results.Add(1) } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) } defer wg.Done() }(it) } - // Links header, first, next, last... linksHeader := linkheader.Parse(resp.Header.Get("Link")) - // Process the next link recursively for _, link := range linksHeader { + select { + case <-ctx.Done(): + return + default: + } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -157,8 +165,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -168,8 +180,9 @@ func (s *Source) AddApiKeys(keys []string) { // Statistics returns the statistics for the source func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go similarity index 59% rename from v2/pkg/subscraping/sources/hackertarget/hackertarget.go rename to pkg/subscraping/sources/hackertarget/hackertarget.go index de20cbbf0..2c00e0188 100644 --- a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -12,9 +12,12 @@ import ( // Source is the passive scraping agent type Source struct { + apiKeys []string timeTaken time.Duration errors int results int + requests int + skipped bool } // Run function returns all subdomains found with the service @@ -22,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -29,7 +33,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain)) + htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + } + + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + + s.requests++ + resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -37,18 +50,32 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } match := session.Extractor.Extract(line) for _, subdomain := range match { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() @@ -69,12 +96,16 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { @@ -82,5 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go similarity index 81% rename from v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go rename to pkg/subscraping/sources/hudsonrock/hudsonrock.go index b109a3192..79eca0690 100644 --- a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -26,6 +26,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -47,21 +50,25 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response hudsonrockResponse err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) { for _, subdomain := range session.Extractor.Extract(record.URL) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } @@ -83,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -95,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/v2/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go similarity index 84% rename from v2/pkg/subscraping/sources/intelx/intelx.go rename to pkg/subscraping/sources/intelx/intelx.go index c08b3ff3c..560f4fe6b 100644 --- a/v2/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -56,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -85,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -98,15 +101,21 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) status := 0 for status == 0 || status == 3 { + select { + case <-ctx.Done(): + return + default: + } + s.requests++ resp, err = session.Get(ctx, resultsURL, "", nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -119,7 +128,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -127,17 +136,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) status = response.Status for _, hostname := range response.Selectors { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}: + s.results++ } - s.results++ } } }() @@ -158,8 +169,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -172,6 +187,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go similarity index 84% rename from v2/pkg/subscraping/sources/leakix/leakix.go rename to pkg/subscraping/sources/leakix/leakix.go index 7061ccd11..84c07b4d0 100644 --- a/v2/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,13 +42,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers["api-key"] = randomApiKey } // Request + s.requests++ resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } - defer resp.Body.Close() + + defer session.DiscardHTTPResponse(resp) + if resp.StatusCode != 200 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} s.errors++ @@ -62,10 +67,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, result := range subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: + s.results++ } - s.results++ } }() return results @@ -84,8 +91,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -96,6 +107,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go new file mode 100644 index 000000000..b27f689da --- /dev/null +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -0,0 +1,174 @@ +// Package merklemap logic +package merklemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/url" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + // Pick an API key, skip if no key is found + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // Default headers + headers := map[string]string{ + "accept": "application/json", + // Set a user agent to prevent random one from pkg/subscraping/agent.go, it triggers the cloudflare protection of the api + "User-Agent": "subfinder", + "Authorization": "Bearer " + randomApiKey, + } + + // Fetch all pages with pagination + // https://www.merklemap.com/documentation/search + s.fetchAllPages(ctx, domain, headers, session, results) + }() + return results +} + +// fetchAllPages fetches all pages of results using pagination +func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + baseURL := "https://api.merklemap.com/v1/search?query=" + url.QueryEscape("*."+domain) + totalCount := math.MaxInt + processedResults := 0 + + // Iterate through all pages + for page := 0; processedResults < totalCount; page++ { + pageResp, err := s.fetchPage(ctx, baseURL, page, headers, session) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if page == 0 { + totalCount = pageResp.Count + } + + // Stop if this page returned no results + if len(pageResp.Results) == 0 { + break + } + + for _, result := range pageResp.Results { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: result.Hostname, + } + s.results++ + processedResults++ + } + + } +} + +// fetchPage fetches a single page of results +func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) { + url := baseURL + "&page=" + strconv.Itoa(page) + + s.requests++ + resp, err := session.Get(ctx, url, "", headers) + if err != nil { + return nil, err + } + defer session.DiscardHTTPResponse(resp) + + if resp.StatusCode != 200 { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, err) + } + return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, string(respBody)) + } + + var pageResponse response + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(bytes.NewReader(respBody)) + if err := decoder.Decode(&pageResponse); err != nil { + return nil, err + } + + return &pageResponse, nil +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "merklemap" +} + +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} + +type response struct { + Count int `json:"count"` + Results []struct { + Hostname string `json:"hostname"` + SubjectCommonName string `json:"subject_common_name"` + FirstSeen string `json:"first_seen"` + } `json:"results"` +} diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go similarity index 76% rename from v2/pkg/subscraping/sources/netlas/netlas.go rename to pkg/subscraping/sources/netlas/netlas.go index 5df10c753..fbf65e55f 100644 --- a/v2/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -40,6 +40,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -47,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,7 +65,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ + s.requests++ + resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, }, nil, subscraping.BasicAuth{}) @@ -72,14 +75,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return - } else if resp.StatusCode != 200 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + } else if resp1.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp1.StatusCode)} s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp1.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp1.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ @@ -98,10 +106,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Make a single POST request to get all domains via download method apiUrl := "https://app.netlas.io/api/domains/download/" - query := fmt.Sprintf("domain:(domain:*.%s AND NOT domain:%s)", domain, domain) - requestBody := map[string]interface{}{ - "q": query, - "fields": []string{"*"}, + query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) + requestBody := map[string]any{ + "q": query, + "fields": []string{"*"}, "source_type": "include", "size": domainsCount.Count, } @@ -115,25 +123,31 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ - "accept": "application/json", - "X-API-Key": randomApiKey, + s.requests++ + resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ + "accept": "application/json", + "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } - defer resp.Body.Close() - body, err = io.ReadAll(resp.Body) + defer func() { + if err := resp2.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + body, err = io.ReadAll(resp2.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ return } - if resp.StatusCode == 429 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + if resp2.StatusCode == 429 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp2.StatusCode)} s.errors++ return } @@ -148,10 +162,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, item := range data { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain}: + s.results++ } - s.results++ } }() @@ -172,8 +188,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -184,6 +204,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/onyphe/onyphe.go b/pkg/subscraping/sources/onyphe/onyphe.go new file mode 100644 index 000000000..8c58dd457 --- /dev/null +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -0,0 +1,318 @@ +// Package onyphe logic +package onyphe + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type OnypheResponse struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Total int `json:"total"` + MaxPage int `json:"max_page"` +} + +type Result struct { + Subdomains []string `json:"subdomains"` + Hostname string `json:"hostname"` + Forward string `json:"forward"` + Reverse string `json:"reverse"` + Host string `json:"host"` + Domain string `json:"domain"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} + + page := 1 + pageSize := 1000 + + for { + select { + case <-ctx.Done(): + return + default: + } + var resp *http.Response + var err error + + urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", + url.QueryEscape("category:resolver domain:"+domain), page, pageSize) + s.requests++ + resp, err = session.Get(ctx, urlWithQuery, "", headers) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var respOnyphe OnypheResponse + err = json.NewDecoder(resp.Body).Decode(&respOnyphe) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + session.DiscardHTTPResponse(resp) + + for _, record := range respOnyphe.Results { + select { + case <-ctx.Done(): + return + default: + } + for _, subdomain := range record.Subdomains { + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + + if record.Hostname != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } + + if record.Forward != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Forward} + s.results++ + } + + if record.Reverse != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Reverse} + s.results++ + } + } + + if len(respOnyphe.Results) == 0 || page >= respOnyphe.MaxPage { + break + } + + page++ + + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "onyphe" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} + +type OnypheResponseRaw struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page json.RawMessage `json:"page"` + PageSize json.RawMessage `json:"page_size"` + Total json.RawMessage `json:"total"` + MaxPage json.RawMessage `json:"max_page"` +} + +func (o *OnypheResponse) UnmarshalJSON(data []byte) error { + var raw OnypheResponseRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + o.Error = raw.Error + o.Results = raw.Results + + if pageStr := string(raw.Page); pageStr != "" { + if page, err := strconv.Atoi(pageStr); err == nil { + o.Page = page + } else { + var pageStrQuoted string + if err := json.Unmarshal(raw.Page, &pageStrQuoted); err == nil { + if page, err := strconv.Atoi(pageStrQuoted); err == nil { + o.Page = page + } + } + } + } + + if pageSizeStr := string(raw.PageSize); pageSizeStr != "" { + if pageSize, err := strconv.Atoi(pageSizeStr); err == nil { + o.PageSize = pageSize + } else { + var pageSizeStrQuoted string + if err := json.Unmarshal(raw.PageSize, &pageSizeStrQuoted); err == nil { + if pageSize, err := strconv.Atoi(pageSizeStrQuoted); err == nil { + o.PageSize = pageSize + } + } + } + } + + if totalStr := string(raw.Total); totalStr != "" { + if total, err := strconv.Atoi(totalStr); err == nil { + o.Total = total + } else { + var totalStrQuoted string + if err := json.Unmarshal(raw.Total, &totalStrQuoted); err == nil { + if total, err := strconv.Atoi(totalStrQuoted); err == nil { + o.Total = total + } + } + } + } + + if maxPageStr := string(raw.MaxPage); maxPageStr != "" { + if maxPage, err := strconv.Atoi(maxPageStr); err == nil { + o.MaxPage = maxPage + } else { + var maxPageStrQuoted string + if err := json.Unmarshal(raw.MaxPage, &maxPageStrQuoted); err == nil { + if maxPage, err := strconv.Atoi(maxPageStrQuoted); err == nil { + o.MaxPage = maxPage + } + } + } + } + + return nil +} + +type ResultRaw struct { + Subdomains json.RawMessage `json:"subdomains"` + Hostname json.RawMessage `json:"hostname"` + Forward json.RawMessage `json:"forward"` + Reverse json.RawMessage `json:"reverse"` + Host json.RawMessage `json:"host"` + Domain json.RawMessage `json:"domain"` +} + +func (r *Result) UnmarshalJSON(data []byte) error { + var raw ResultRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + var subdomains []string + if err := json.Unmarshal(raw.Subdomains, &subdomains); err == nil { + r.Subdomains = subdomains + } else { + var subdomainStr string + if err := json.Unmarshal(raw.Subdomains, &subdomainStr); err == nil { + r.Subdomains = []string{subdomainStr} + } + } + + if len(raw.Hostname) > 0 { + var hostnameStr string + if err := json.Unmarshal(raw.Hostname, &hostnameStr); err == nil { + r.Hostname = hostnameStr + } else { + var hostnameArr []string + if err := json.Unmarshal(raw.Hostname, &hostnameArr); err == nil && len(hostnameArr) > 0 { + r.Hostname = hostnameArr[0] + } + } + } + + if len(raw.Forward) > 0 { + _ = json.Unmarshal(raw.Forward, &r.Forward) + } + + if len(raw.Reverse) > 0 { + _ = json.Unmarshal(raw.Reverse, &r.Reverse) + } + + if len(raw.Host) > 0 { + var hostStr string + if err := json.Unmarshal(raw.Host, &hostStr); err == nil { + r.Host = hostStr + } else { + var hostArr []string + if err := json.Unmarshal(raw.Host, &hostArr); err == nil && len(hostArr) > 0 { + r.Host = hostArr[0] + } + } + } + + if len(raw.Domain) > 0 { + var domainStr string + if err := json.Unmarshal(raw.Domain, &domainStr); err == nil { + r.Domain = domainStr + } else { + var domainArr []string + if err := json.Unmarshal(raw.Domain, &domainArr); err == nil && len(domainArr) > 0 { + r.Domain = domainArr[0] + } + } + } + + return nil +} diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go new file mode 100644 index 000000000..09cd70eac --- /dev/null +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -0,0 +1,135 @@ +// Package profundis logic +package profundis + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "strings" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + requestBody, err := json.Marshal(map[string]string{"domain": domain}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + headers := map[string]string{ + "Content-Type": "application/json", + "X-API-KEY": randomApiKey, + "Accept": "text/event-stream", + } + + s.requests++ + resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "", + headers, bytes.NewReader(requestBody)) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: line}: + s.results++ + } + } + + if err := scanner.Err(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + return results +} + +func (s *Source) Name() string { + return "profundis" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go new file mode 100644 index 000000000..8c0aae962 --- /dev/null +++ b/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -0,0 +1,163 @@ +// Package pugrecon logic +package pugrecon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// pugreconResult stores a single result from the pugrecon API +type pugreconResult struct { + Name string `json:"name"` +} + +// pugreconAPIResponse stores the response from the pugrecon API +type pugreconAPIResponse struct { + Results []pugreconResult `json:"results"` + QuotaRemaining int `json:"quota_remaining"` + Limited bool `json:"limited"` + TotalResults int `json:"total_results"` + Message string `json:"message"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // Prepare POST request data + postData := map[string]string{"domain_name": domain} + bodyBytes, err := json.Marshal(postData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)} + s.errors++ + return + } + bodyReader := bytes.NewReader(bodyBytes) + + // Prepare headers + headers := map[string]string{ + "Authorization": "Bearer " + randomApiKey, + "Content-Type": "application/json", + "Accept": "application/json", + } + + apiURL := "https://pugrecon.com/api/v1/domains" + s.requests++ + resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to close response body: %w", err)} + s.errors++ + } + }() + + if resp.StatusCode != http.StatusOK { + errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode) + // Attempt to read error message from body if possible + var apiResp pugreconAPIResponse + if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { + errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message) + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", errorMsg)} + s.errors++ + return + } + + var response pugreconAPIResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range response.Results { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name}: + s.results++ + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "pugrecon" +} + +// IsDefault returns false as this is not a default source. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport returns false as this source does not support recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// KeyRequirement returns the API key requirement level for this source. +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +// NeedsKey returns true as this source requires an API key. +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +// AddApiKeys adds the API keys for the source. +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns the statistics for the source. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go new file mode 100644 index 000000000..6ad6bee7e --- /dev/null +++ b/pkg/subscraping/sources/quake/quake.go @@ -0,0 +1,165 @@ +// Package quake logic +package quake + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type quakeResults struct { + Code int `json:"code"` + Message string `json:"message"` + Data []struct { + Service struct { + HTTP struct { + Host string `json:"host"` + } `json:"http"` + } + } `json:"data"` + Meta struct { + Pagination struct { + Total int `json:"total"` + } `json:"pagination"` + } `json:"meta"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // quake api doc https://quake.360.cn/quake/#/help + var pageSize = 500 + var start = 0 + var totalResults = -1 + + for { + select { + case <-ctx.Done(): + return + default: + } + var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + s.requests++ + resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ + "Content-Type": "application/json", "X-QuakeToken": randomApiKey, + }, bytes.NewReader(requestBody)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var response quakeResults + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + session.DiscardHTTPResponse(resp) + + if response.Code != 0 { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + } + s.errors++ + return + } + + if totalResults == -1 { + totalResults = response.Meta.Pagination.Total + } + + for _, quakeDomain := range response.Data { + select { + case <-ctx.Done(): + return + default: + } + subdomain := quakeDomain.Service.HTTP.Host + if strings.ContainsAny(subdomain, "ζš‚ζ— ζƒι™") { + continue + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + + if len(response.Data) == 0 || start+pageSize >= totalResults { + break + } + + start += pageSize + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "quake" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go similarity index 80% rename from v2/pkg/subscraping/sources/rapiddns/rapiddns.go rename to pkg/subscraping/sources/rapiddns/rapiddns.go index 5daae6153..e2e679a41 100644 --- a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -19,6 +19,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -36,6 +38,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 maxPages := 1 for { + select { + case <-ctx.Done(): + return + default: + } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -48,16 +56,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } if maxPages == 1 { @@ -93,8 +105,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -106,5 +122,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go similarity index 82% rename from v2/pkg/subscraping/sources/reconcloud/reconcloud.go rename to pkg/subscraping/sources/reconcloud/reconcloud.go index 5a638b5f4..d572d525c 100644 --- a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -56,15 +59,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response.CloudAssetsList) > 0 { for _, cloudAsset := range response.CloudAssetsList { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}: + s.results++ + } } } }() @@ -85,8 +92,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -98,5 +109,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconeer/reconeer.go b/pkg/subscraping/sources/reconeer/reconeer.go new file mode 100644 index 000000000..d0e905be7 --- /dev/null +++ b/pkg/subscraping/sources/reconeer/reconeer.go @@ -0,0 +1,116 @@ +package reconeer + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Subdomains []subdomain `json:"subdomains"` +} + +type subdomain struct { + Subdomain string `json:"subdomain"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + headers := map[string]string{ + "Accept": "application/json", + } + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["X-API-KEY"] = randomApiKey + } + apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) + s.requests++ + resp, err := session.Get(ctx, apiURL, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + defer session.DiscardHTTPResponse(resp) + + if resp.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} + s.errors++ + return + } + var responseData response + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&responseData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + for _, result := range responseData.Subdomains { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: + s.results++ + } + } + }() + return results +} + +func (s *Source) Name() string { + return "reconeer" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go similarity index 81% rename from v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go rename to pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index ecbdcace2..825161eed 100644 --- a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 pageSize := 1000 go func() { defer func(startTime time.Time) { @@ -56,6 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1] requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"} getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -67,16 +70,22 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.errors++ return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Metadata.ResultCount > pageSize { totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize for page := 1; page <= totalPages; page++ { + select { + case <-ctx.Done(): + return + default: + } getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -88,22 +97,30 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.errors++ continue } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } } else { for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } @@ -123,8 +140,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -137,5 +158,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go similarity index 74% rename from v2/pkg/subscraping/sources/riddler/riddler.go rename to pkg/subscraping/sources/riddler/riddler.go index 0afe725eb..142fc6f37 100644 --- a/v2/pkg/subscraping/sources/riddler/riddler.go +++ b/pkg/subscraping/sources/riddler/riddler.go @@ -15,6 +15,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -22,6 +23,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -29,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -39,16 +42,27 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } line := scanner.Text() if line == "" { continue } for _, subdomain := range session.Extractor.Extract(line) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) }() return results @@ -67,8 +81,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -80,5 +98,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/sources/robtex/robtext.go b/pkg/subscraping/sources/robtex/robtext.go similarity index 87% rename from v2/pkg/subscraping/sources/robtex/robtext.go rename to pkg/subscraping/sources/robtex/robtext.go index 5d130a42d..25f4e0418 100644 --- a/v2/pkg/subscraping/sources/robtex/robtext.go +++ b/pkg/subscraping/sources/robtex/robtext.go @@ -62,6 +62,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, result := range ips { + select { + case <-ctx.Done(): + return + default: + } if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers) if err != nil { @@ -70,8 +75,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, result := range domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}: + s.results++ + } } } } @@ -89,6 +98,8 @@ func enumerate(ctx context.Context, session *subscraping.Session, targetURL stri return results, err } + defer session.DiscardHTTPResponse(resp) + scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() @@ -104,8 +115,6 @@ func enumerate(ctx context.Context, session *subscraping.Session, targetURL stri results = append(results, response) } - resp.Body.Close() - return results, nil } @@ -122,8 +131,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go new file mode 100644 index 000000000..577ff0b30 --- /dev/null +++ b/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -0,0 +1,134 @@ +package rsecloud + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Count int `json:"count"` + Data []string `json:"data"` + Page int `json:"page"` + PageSize int `json:"pagesize"` + TotalPages int `json:"total_pages"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "X-API-Key": randomApiKey} + + fetchSubdomains := func(endpoint string) { + page := 1 + for { + select { + case <-ctx.Done(): + return + default: + } + s.requests++ + resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var rseCloudResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&rseCloudResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range rseCloudResponse.Data { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + + if page >= rseCloudResponse.TotalPages { + break + } + page++ + } + } + + fetchSubdomains("active") + fetchSubdomains("passive") + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "rsecloud" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go similarity index 81% rename from v2/pkg/subscraping/sources/securitytrails/securitytrails.go rename to pkg/subscraping/sources/securitytrails/securitytrails.go index e254d9c20..c852b52f5 100644 --- a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -31,6 +31,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -56,18 +58,26 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} for { + select { + case <-ctx.Done(): + return + default: + } var resp *http.Response var err error if scrollId == "" { - var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain)) + var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) + s.requests++ resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) } if err != nil && ptr.Safe(resp).StatusCode == 403 { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) } @@ -83,18 +93,27 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range securityTrailsResponse.Records { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: + s.results++ + } } for _, subdomain := range securityTrailsResponse.Subdomains { + select { + case <-ctx.Done(): + return + default: + } if strings.HasSuffix(subdomain, ".") { subdomain += domain } else { @@ -128,8 +147,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -142,5 +165,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go similarity index 86% rename from v2/pkg/subscraping/sources/shodan/shodan.go rename to pkg/subscraping/sources/shodan/shodan.go index 8d4925d79..f5f3d365c 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -48,15 +50,21 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 for { + select { + case <-ctx.Done(): + return + default: + } searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) @@ -75,6 +83,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, data := range response.Subdomains { + select { + case <-ctx.Done(): + return + default: + } value := fmt.Sprintf("%s.%s", data, response.Domain) results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: value, @@ -105,8 +118,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -117,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go similarity index 83% rename from v2/pkg/subscraping/sources/sitedossier/sitedossier.go rename to pkg/subscraping/sources/sitedossier/sitedossier.go index dacc3c66b..560a0bda3 100644 --- a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba default: } + s.requests++ resp, err := session.SimpleGet(ctx, baseURL) isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound if err != nil && !isnotfound { @@ -63,15 +66,19 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba if err != nil { results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } match := reNext.FindStringSubmatch(src) @@ -93,8 +100,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -106,5 +117,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go new file mode 100644 index 000000000..6edd2983a --- /dev/null +++ b/pkg/subscraping/sources/thc/thc.go @@ -0,0 +1,135 @@ +// Package thc logic +package thc + +import ( + "bytes" + "context" + "encoding/json" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Domains []struct { + Domain string `json:"domain"` + } `json:"domains"` + NextPageState string `json:"next_page_state"` +} + +// Source is the passive scraping agent +type Source struct { + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +type requestBody struct { + Domain string `json:"domain"` + PageState string `json:"page_state"` + Limit int `json:"limit"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + var pageState string + headers := map[string]string{"Content-Type": "application/json"} + apiURL := "https://ip.thc.org/api/v1/lookup/subdomains" + + for { + reqBody := requestBody{ + Domain: domain, + PageState: pageState, + Limit: 1000, + } + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + s.requests++ + resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var thcResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&thcResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, domainRecord := range thcResponse.Domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domainRecord.Domain} + s.results++ + } + + pageState = thcResponse.NextPageState + + if pageState == "" { + break + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "thc" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(_ []string) { + // No API keys needed for THC +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/v2/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go similarity index 85% rename from v2/pkg/subscraping/sources/threatbook/threatbook.go rename to pkg/subscraping/sources/threatbook/threatbook.go index 1c10a8d10..3871920a3 100644 --- a/v2/pkg/subscraping/sources/threatbook/threatbook.go +++ b/pkg/subscraping/sources/threatbook/threatbook.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -51,6 +53,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -64,10 +67,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.ResponseCode != 0 { results <- subscraping.Result{ @@ -87,8 +90,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if total > 0 { for _, subdomain := range response.Data.SubDomains.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() @@ -109,8 +116,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -123,5 +134,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go new file mode 100644 index 000000000..897c73345 --- /dev/null +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -0,0 +1,135 @@ +package threatcrowd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// threatCrowdResponse represents the JSON response from the ThreatCrowd API. +type threatCrowdResponse struct { + ResponseCode string `json:"response_code"` + Subdomains []string `json:"subdomains"` + Undercount string `json:"undercount"` +} + +// Source implements the subscraping.Source interface for ThreatCrowd. +type Source struct { + timeTaken time.Duration + errors int + results int + requests int +} + +// Run queries the ThreatCrowd API for the given domain and returns found subdomains. +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func(startTime time.Time) { + defer func() { + s.timeTaken = time.Since(startTime) + close(results) + }() + + url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + s.requests++ + resp, err := session.Client.Do(req) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + if resp.StatusCode != http.StatusOK { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} + s.errors++ + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + var tcResponse threatCrowdResponse + if err := json.Unmarshal(body, &tcResponse); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range tcResponse.Subdomains { + if subdomain != "" { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + }(time.Now()) + + return results +} + +// Name returns the name of the source. +func (s *Source) Name() string { + return "threatcrowd" +} + +// IsDefault indicates whether this source is enabled by default. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport indicates if the source supports recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// KeyRequirement returns the API key requirement level for this source. +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + +// NeedsKey indicates if the source requires an API key. +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +// AddApiKeys is a no-op since ThreatCrowd does not require an API key. +func (s *Source) AddApiKeys(_ []string) {} + +// Statistics returns usage statistics. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Requests: s.requests, + } +} diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go similarity index 81% rename from v2/pkg/subscraping/sources/threatminer/threatminer.go rename to pkg/subscraping/sources/threatminer/threatminer.go index adffabd89..d7224c8e3 100644 --- a/v2/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -36,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -44,7 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) @@ -55,8 +58,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range data.Results { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() @@ -76,8 +83,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -89,5 +100,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go new file mode 100644 index 000000000..1b0a9fcff --- /dev/null +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -0,0 +1,215 @@ +// Package urlscan logic +package urlscan + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the URLScan API base URL + baseURL = "https://urlscan.io/api/v1/search/" + // maxPages is the maximum number of pages to fetch + maxPages = 5 + // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) + maxPerPage = 100 +) + +// response represents the URLScan API response structure +type response struct { + Results []struct { + Task struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"task"` + Page struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"page"` + Sort []interface{} `json:"sort"` + } `json:"results"` + HasMore bool `json:"has_more"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + s.skipped = false + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"api-key": randomApiKey} + + // Search with wildcard to get more subdomain results + s.enumerate(ctx, domain, headers, session, results) + }() + + return results +} + +// enumerate performs the actual enumeration with pagination +func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + var searchAfter string + currentPage := 0 + + for { + select { + case <-ctx.Done(): + return + default: + } + + if currentPage >= maxPages { + break + } + + // Build search URL + searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) + if searchAfter != "" { + searchURL += "&search_after=" + url.QueryEscape(searchAfter) + } + + s.requests++ + resp, err := session.Get(ctx, searchURL, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + session.DiscardHTTPResponse(resp) + + // Process results - extract subdomains from multiple fields + for _, result := range data.Results { + candidates := []string{ + result.Task.Domain, + result.Page.Domain, + } + + // Also extract from URLs if present + if result.Task.URL != "" { + if u, err := url.Parse(result.Task.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + if result.Page.URL != "" { + if u, err := url.Parse(result.Page.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + + for _, candidate := range candidates { + if candidate == "" { + continue + } + for _, subdomain := range session.Extractor.Extract(candidate) { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + } + + // Check pagination conditions + if !data.HasMore || len(data.Results) == 0 { + break + } + + // Get sort value for next page + lastResult := data.Results[len(data.Results)-1] + if len(lastResult.Sort) == 0 { + break + } + + // Build search_after parameter + sortValues := make([]string, len(lastResult.Sort)) + for i, v := range lastResult.Sort { + switch val := v.(type) { + case float64: + sortValues[i] = fmt.Sprintf("%.0f", val) + default: + sortValues[i] = fmt.Sprintf("%v", v) + } + } + searchAfter = strings.Join(sortValues, ",") + currentPage++ + } +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "urlscan" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go similarity index 74% rename from v2/pkg/subscraping/sources/virustotal/virustotal.go rename to pkg/subscraping/sources/virustotal/virustotal.go index f996a1628..812c7c9cf 100644 --- a/v2/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -49,12 +51,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if randomApiKey == "" { return } - var cursor string = "" + var cursor = "" for { - var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain) + select { + case <-ctx.Done(): + return + default: + } + var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } + s.requests++ resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -62,7 +70,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) @@ -73,8 +86,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range data.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}: + s.results++ + } } cursor = data.Meta.Cursor if cursor == "" { @@ -99,8 +116,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -111,6 +132,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go similarity index 79% rename from v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go rename to pkg/subscraping/sources/waybackarchive/waybackarchive.go index 5edc78382..25687659c 100644 --- a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -39,23 +42,31 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { - // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() @@ -76,8 +87,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -89,5 +104,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go similarity index 82% rename from v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go rename to pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index eafd340af..284b920a6 100644 --- a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -33,6 +33,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -41,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -54,6 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -67,15 +70,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range data.Result.Records { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: + s.results++ + } } }() @@ -95,8 +102,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -109,5 +120,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go new file mode 100644 index 000000000..36a27dc6d --- /dev/null +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -0,0 +1,162 @@ +// Package windvane logic +package windvane + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data responseData `json:"data"` +} + +type responseData struct { + List []domainEntry `json:"list"` + PageResponse pageInfo `json:"page_response"` +} + +type domainEntry struct { + Domain string `json:"domain"` +} + +type pageInfo struct { + Total string `json:"total"` + Count string `json:"count"` + TotalPage string `json:"total_page"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "X-Api-Key": randomApiKey} + + page := 1 + count := 1000 + for { + select { + case <-ctx.Done(): + return + default: + } + var resp *http.Response + var err error + + requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}}) + s.requests++ + resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", + "", headers, bytes.NewReader(requestBody)) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer session.DiscardHTTPResponse(resp) + + var windvaneResponse response + err = json.NewDecoder(resp.Body).Decode(&windvaneResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + for _, record := range windvaneResponse.Data.List { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: + s.results++ + } + } + + pageInfo := windvaneResponse.Data.PageResponse + var totalRecords, recordsPerPage int + + if totalRecords, err = strconv.Atoi(pageInfo.Total); err != nil { + break + } + if recordsPerPage, err = strconv.Atoi(pageInfo.Count); err != nil { + break + } + + if (page-1)*recordsPerPage >= totalRecords { + break + } + + page++ + } + + }() + + return results +} + +func (s *Source) Name() string { + return "windvane" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go similarity index 85% rename from v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go rename to pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 62c2d1f91..61aa7197a 100644 --- a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,7 +65,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var pages = 1 for currentPage := 1; currentPage <= pages; currentPage++ { + select { + case <-ctx.Done(): + return + default: + } api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) + s.requests++ resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil { @@ -87,8 +95,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se _ = resp.Body.Close() pages = int(res.Total/1000) + 1 for _, r := range res.List { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}: + s.results++ + } } } }() @@ -109,8 +121,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -123,5 +139,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/v2/pkg/subscraping/types.go b/pkg/subscraping/types.go similarity index 77% rename from v2/pkg/subscraping/types.go rename to pkg/subscraping/types.go index 454ad776f..381187171 100644 --- a/v2/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -16,7 +16,8 @@ const ( ) type CustomRateLimit struct { - Custom mapsutil.SyncLockMap[string, uint] + Custom mapsutil.SyncLockMap[string, uint] + CustomDuration mapsutil.SyncLockMap[string, time.Duration] } // BasicAuth request's Authorization header @@ -28,11 +29,21 @@ type BasicAuth struct { // Statistics contains statistics about the scraping process type Statistics struct { TimeTaken time.Duration + Requests int Errors int Results int Skipped bool } +// KeyRequirement represents the API key requirement level for a source +type KeyRequirement int + +const ( + NoKey KeyRequirement = iota + OptionalKey + RequiredKey +) + // Source is an interface inherited by each passive source type Source interface { // Run takes a domain as argument and a session object @@ -52,7 +63,11 @@ type Source interface { // not just root domains. HasRecursiveSupport() bool - // NeedsKey returns true if the source requires an API key + // KeyRequirement returns the API key requirement level for this source + KeyRequirement() KeyRequirement + + // NeedsKey returns true if the source requires an API key. + // Deprecated: Use KeyRequirement() instead for more granular control. NeedsKey() bool AddApiKeys([]string) @@ -75,6 +90,8 @@ type Session struct { Client *http.Client // Rate limit instance MultiRateLimiter *ratelimit.MultiLimiter + // Timeout is the timeout in seconds for requests + Timeout int } // Result is a result structure returned by a source diff --git a/v2/pkg/subscraping/utils.go b/pkg/subscraping/utils.go similarity index 100% rename from v2/pkg/subscraping/utils.go rename to pkg/subscraping/utils.go diff --git a/v2/pkg/testutils/integration.go b/pkg/testutils/integration.go similarity index 91% rename from v2/pkg/testutils/integration.go rename to pkg/testutils/integration.go index 2358cad20..d18230e95 100644 --- a/v2/pkg/testutils/integration.go +++ b/pkg/testutils/integration.go @@ -27,8 +27,8 @@ func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]st return nil, err } var parts []string - items := strings.Split(string(data), "\n") - for _, i := range items { + items := strings.SplitSeq(string(data), "\n") + for i := range items { if i != "" { parts = append(parts, i) } diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml deleted file mode 100644 index 6e97216d6..000000000 --- a/v2/.goreleaser.yml +++ /dev/null @@ -1,43 +0,0 @@ -before: - hooks: - - go mod tidy - -builds: -- env: - - CGO_ENABLED=0 - goos: - - windows - - linux - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 - - ignore: - - goos: darwin - goarch: '386' - - goos: windows - goarch: 'arm' - - binary: '{{ .ProjectName }}' - main: cmd/subfinder/main.go - -archives: -- format: zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' - -checksum: - algorithm: sha256 - -announce: - slack: - enabled: true - channel: '#release' - username: GoReleaser - message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' - - discord: - enabled: true - message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go deleted file mode 100644 index d798fe702..000000000 --- a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go +++ /dev/null @@ -1,194 +0,0 @@ -// Package binaryedge logic -package binaryedge - -import ( - "context" - "errors" - "fmt" - "math" - "net/url" - "strconv" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -const ( - v1 = "v1" - v2 = "v2" - baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" - v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" - v1PageSizeParam = "pagesize" - pageParam = "page" - firstPage = 1 - maxV1PageSize = 10000 -) - -type subdomainsResponse struct { - Message string `json:"message"` - Title string `json:"title"` - Status interface{} `json:"status"` // string for v1, int for v2 - Subdomains []string `json:"events"` - Page int `json:"page"` - PageSize int `json:"pagesize"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []string - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } - - var baseURL string - - authHeader := map[string]string{"X-Key": randomApiKey} - - if isV2(ctx, session, authHeader) { - baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) - } else { - authHeader = map[string]string{"X-Token": randomApiKey} - v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - baseURL = v1URLWithPageSize.String() - } - - if baseURL == "" { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL"), - } - s.errors++ - return - } - - s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) - }() - return results -} - -func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { - pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - - resp, err := session.Get(ctx, pageURL.String(), "", authHeader) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var response subdomainsResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - - // Check error messages - if response.Message != "" && response.Status != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New(response.Message)} - s.errors++ - return - } - - resp.Body.Close() - - for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - - totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) - nextPage := response.Page + 1 - if nextPage <= totalPages { - s.enumerate(ctx, session, baseURL, nextPage, authHeader, results) - } -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "binaryedge" -} - -func (s *Source) IsDefault() bool { - return false -} - -func (s *Source) HasRecursiveSupport() bool { - return true -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} - -func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { - resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) - if err != nil { - session.DiscardHTTPResponse(resp) - return false - } - - resp.Body.Close() - - return true -} - -func addURLParam(targetURL, name, value string) (*url.URL, error) { - u, err := url.Parse(targetURL) - if err != nil { - return u, err - } - q, _ := url.ParseQuery(u.RawQuery) - q.Add(name, value) - u.RawQuery = q.Encode() - - return u, nil -} diff --git a/v2/pkg/subscraping/sources/censys/censys.go b/v2/pkg/subscraping/sources/censys/censys.go deleted file mode 100644 index 04ebe48e5..000000000 --- a/v2/pkg/subscraping/sources/censys/censys.go +++ /dev/null @@ -1,182 +0,0 @@ -// Package censys logic -package censys - -import ( - "context" - "strconv" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - urlutil "github.com/projectdiscovery/utils/url" -) - -const ( - maxCensysPages = 10 - maxPerPage = 100 -) - -type response struct { - Code int `json:"code"` - Status string `json:"status"` - Result result `json:"result"` -} - -type result struct { - Query string `json:"query"` - Total float64 `json:"total"` - DurationMS int `json:"duration_ms"` - Hits []hit `json:"hits"` - Links links `json:"links"` -} - -type hit struct { - Parsed parsed `json:"parsed"` - Names []string `json:"names"` - FingerprintSha256 string `json:"fingerprint_sha256"` -} - -type parsed struct { - ValidityPeriod validityPeriod `json:"validity_period"` - SubjectDN string `json:"subject_dn"` - IssuerDN string `json:"issuer_dn"` -} - -type validityPeriod struct { - NotAfter string `json:"not_after"` - NotBefore string `json:"not_before"` -} - -type links struct { - Next string `json:"next"` - Prev string `json:"prev"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []apiKey - timeTaken time.Duration - errors int - results int - skipped bool -} - -type apiKey struct { - token string - secret string -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey.token == "" || randomApiKey.secret == "" { - s.skipped = true - return - } - - certSearchEndpoint := "https://search.censys.io/api/v2/certificates/search" - cursor := "" - currentPage := 1 - for { - certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - - certSearchEndpointUrl.Params.Add("q", domain) - certSearchEndpointUrl.Params.Add("per_page", strconv.Itoa(maxPerPage)) - if cursor != "" { - certSearchEndpointUrl.Params.Add("cursor", cursor) - } - - resp, err := session.HTTPRequest( - ctx, - "GET", - certSearchEndpointUrl.String(), - "", - nil, - nil, - subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret}, - ) - - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var censysResponse response - err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - - resp.Body.Close() - - for _, hit := range censysResponse.Result.Hits { - for _, name := range hit.Names { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name} - s.results++ - } - } - - // Exit the censys enumeration if last page is reached - cursor = censysResponse.Result.Links.Next - if cursor == "" || currentPage >= maxCensysPages { - break - } - currentPage++ - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "censys" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - return apiKey{k, v} - }) -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs.go b/v2/pkg/subscraping/sources/facebook/ctlogs.go deleted file mode 100644 index 3f19af736..000000000 --- a/v2/pkg/subscraping/sources/facebook/ctlogs.go +++ /dev/null @@ -1,196 +0,0 @@ -package facebook - -import ( - "context" - "encoding/json" - "fmt" - "io" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - errorutil "github.com/projectdiscovery/utils/errors" - "github.com/projectdiscovery/utils/generic" - urlutil "github.com/projectdiscovery/utils/url" -) - -// source: https://developers.facebook.com/tools/ct -// api-docs: https://developers.facebook.com/docs/certificate-transparency-api -// ratelimit: ~20,000 req/hour per appID https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ - -var ( - domainsPerPage = "1000" - authUrl = "https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials" - domainsUrl = "https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=%s&limit=" + domainsPerPage -) - -type apiKey struct { - AppID string - Secret string - AccessToken string // obtained by calling - // https://graph.facebook.com/oauth/access_token?client_id=APP_ID&client_secret=APP_SECRET&grant_type=client_credentials - Error error // error while fetching access token -} - -// FetchAccessToken fetches the access token for the api key -// using app id and secret -func (k *apiKey) FetchAccessToken() { - if generic.EqualsAny("", k.AppID, k.Secret) { - k.Error = fmt.Errorf("invalid app id or secret") - return - } - resp, err := retryablehttp.Get(fmt.Sprintf(authUrl, k.AppID, k.Secret)) - if err != nil { - k.Error = err - return - } - defer resp.Body.Close() - bin, err := io.ReadAll(resp.Body) - if err != nil { - k.Error = err - return - } - auth := &authResponse{} - if err := json.Unmarshal(bin, auth); err != nil { - k.Error = err - return - } - if auth.AccessToken == "" { - k.Error = fmt.Errorf("invalid response from facebook got %v", string(bin)) - return - } - k.AccessToken = auth.AccessToken -} - -// IsValid returns true if the api key is valid -func (k *apiKey) IsValid() bool { - return k.AccessToken != "" -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []apiKey - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - if len(s.apiKeys) == 0 { - s.skipped = true - close(results) - return results - } - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - key := subscraping.PickRandom(s.apiKeys, s.Name()) - domainsURL := fmt.Sprintf(domainsUrl, key.AccessToken, domain) - - for { - // unfortunately, this cannot be parllelized since pagination is cursor based - resp, err := session.Get(ctx, domainsURL, "", nil) - if err != nil { - s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - bin, err := io.ReadAll(resp.Body) - if err != nil { - s.errors++ - gologger.Verbose().Msgf("failed to read response body: %s\n", err) - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - resp.Body.Close() - response := &response{} - if err := json.Unmarshal(bin, response); err != nil { - s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))} - return - } - for _, v := range response.Data { - for _, domain := range v.Domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} - s.results++ - } - } - if response.Paging.Next == "" { - break - } - // cursor includes api key so no need to update it - domainsURL = updateParamInURL(response.Paging.Next, "limit", domainsPerPage) - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "facebook" -} - -// IsDefault returns true if the source should be queried by default -func (s *Source) IsDefault() bool { - return true -} - -// accepts subdomains (e.g. subdomain.domain.tld) -// but also returns all SANs for a certificate which may not match the domain -func (s *Source) HasRecursiveSupport() bool { - return true -} - -// NeedsKey returns true if the source requires an API key -func (s *Source) NeedsKey() bool { - return true -} - -// AddApiKeys adds api keys to the source -func (s *Source) AddApiKeys(keys []string) { - allapikeys := subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - apiKey := apiKey{AppID: k, Secret: v} - apiKey.FetchAccessToken() - if apiKey.Error != nil { - gologger.Warning().Msgf("Could not fetch access token for %s: %s\n", k, apiKey.Error) - } - return apiKey - }) - // filter out invalid keys - for _, key := range allapikeys { - if key.IsValid() { - s.apiKeys = append(s.apiKeys, key) - } - } -} - -// Statistics returns the statistics for the source -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} - -func updateParamInURL(url, param, value string) string { - urlx, err := urlutil.Parse(url) - if err != nil { - return url - } - urlx.Params.Set(param, value) - return urlx.String() -} diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go b/v2/pkg/subscraping/sources/facebook/ctlogs_test.go deleted file mode 100644 index 8869e7901..000000000 --- a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package facebook - -import ( - "encoding/json" - "fmt" - "io" - "os" - "strings" - "testing" - - "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/utils/generic" -) - -var ( - fb_API_ID = "$FB_APP_ID" - fb_API_SECRET = "$FB_APP_SECRET" -) - -func TestFacebookSource(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - updateWithEnv(&fb_API_ID) - updateWithEnv(&fb_API_SECRET) - if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) { - t.SkipNow() - } - k := apiKey{ - AppID: fb_API_ID, - Secret: fb_API_SECRET, - } - k.FetchAccessToken() - if k.Error != nil { - t.Fatal(k.Error) - } - - fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken) - resp, err := retryablehttp.Get(fetchURL) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - bin, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - response := &response{} - if err := json.Unmarshal(bin, response); err != nil { - t.Fatal(err) - } - if len(response.Data) == 0 { - t.Fatal("no data found") - } -} - -func updateWithEnv(key *string) { - if key == nil { - return - } - value := *key - if strings.HasPrefix(value, "$") { - *key = os.Getenv(value[1:]) - } -} diff --git a/v2/pkg/subscraping/sources/facebook/types.go b/v2/pkg/subscraping/sources/facebook/types.go deleted file mode 100644 index 7a0fab23c..000000000 --- a/v2/pkg/subscraping/sources/facebook/types.go +++ /dev/null @@ -1,36 +0,0 @@ -package facebook - -type authResponse struct { - AccessToken string `json:"access_token"` -} - -/* -{ - "data": [ - { - "domains": [ - "docs.hackerone.com" - ], - "id": "10056051421102939" - }, - ... - ], - "paging": { - "cursors": { - "before": "MTAwNTYwNTE0MjExMDI5MzkZD", - "after": "Njc0OTczNTA5NTA1MzUxNwZDZD" - }, - "next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD" - } -} -*/ -// example response - -type response struct { - Data []struct { - Domains []string `json:"domains"` - } `json:"data"` - Paging struct { - Next string `json:"next"` - } `json:"paging"` -} diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go deleted file mode 100644 index dc3b77021..000000000 --- a/v2/pkg/subscraping/sources/hunter/hunter.go +++ /dev/null @@ -1,131 +0,0 @@ -package hunter - -import ( - "context" - "encoding/base64" - "fmt" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -type hunterResp struct { - Code int `json:"code"` - Data hunterData `json:"data"` - Message string `json:"message"` -} - -type infoArr struct { - URL string `json:"url"` - IP string `json:"ip"` - Port int `json:"port"` - Domain string `json:"domain"` - Protocol string `json:"protocol"` -} - -type hunterData struct { - InfoArr []infoArr `json:"arr"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []string - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } - - var pages = 1 - for currentPage := 1; currentPage <= pages; currentPage++ { - // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 - qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) - if err != nil && resp == nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var response hunterResp - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - resp.Body.Close() - - if response.Code == 401 || response.Code == 400 { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), - } - s.errors++ - return - } - - if response.Data.Total > 0 { - for _, hunterInfo := range response.Data.InfoArr { - subdomain := hunterInfo.Domain - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - } - pages = int(response.Data.Total/1000) + 1 - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "hunter" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -}