diff --git a/.github/renovate.json b/.github/renovate.json index 319817ef7ced..8e955f8c7eb0 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,8 +1,21 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:base" + "extends": ["github>trufflesecurity/.github:renovate-config#v1.0.4"], + "customManagers": [ + { + "description": "scripts/lint.sh pins the same golangci-lint version as .github/workflows/lint.yml", + "customType": "regex", + "managerFilePatterns": ["scripts/lint.sh"], + "matchStrings": ["GOLANGCI_LINT_VERSION=\"(?v[^\"]+)\""], + "depNameTemplate": "golangci/golangci-lint", + "datasourceTemplate": "github-releases" + } ], - "prConcurrentLimit": 3, - "prHourlyLimit": 2 + "packageRules": [ + { + "description": "Keep the workflow and script golangci-lint pins in one PR", + "matchDepNames": ["golangci/golangci-lint"], + "groupName": "golangci-lint" + } + ] } diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 70e112ab9315..6e7ffe94f192 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,14 +38,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,4 +56,4 @@ jobs: run: | go build . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4 diff --git a/.github/workflows/detector-corpora-test.yml b/.github/workflows/detector-corpora-test.yml new file mode 100644 index 000000000000..9d3a7f19b674 --- /dev/null +++ b/.github/workflows/detector-corpora-test.yml @@ -0,0 +1,266 @@ +name: Corpora Test + +on: + workflow_dispatch: + pull_request: + types: [opened] + paths: + - 'pkg/detectors/**' + - 'pkg/engine/defaults/defaults.go' + - '.github/workflows/detector-corpora-test.yml' + - 'scripts/test/detector_corpora_test.sh' + - 'scripts/test/diff_corpora_results.py' + - 'scripts/test/detect_changed_detectors.sh' + +env: + DATASETS: | + s3://trufflehog-corpora-datasets/contents.2025-11-04.jsonl.zstd + +jobs: + corpora-test: + if: ${{ github.repository == 'trufflesecurity/trufflehog' && !github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout code + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.27" + + - name: Install dependencies + run: sudo apt-get install -y zstd jq + + - name: Resolve merge-base + id: merge_base + shell: bash + run: | + set -o pipefail + git fetch --no-tags --prune origin main + MERGE_BASE=$(git merge-base origin/main HEAD) + echo "Merge base: $MERGE_BASE" + echo "sha=$MERGE_BASE" >> "$GITHUB_OUTPUT" + + # Determine which detectors changed in this PR. The PR build scopes its + # scan to the full set; the main build excludes detectors that don't + # exist there yet (new detectors). If the set is empty, the workflow + # short-circuits with a skip comment — scoping is the entire point of + # Phase 2, falling back to scan-all defeats it. + - name: Detect changed detectors + id: detect + shell: bash + env: + BASE_REF: ${{ steps.merge_base.outputs.sha }} + run: | + set -o pipefail + chmod +x scripts/test/detect_changed_detectors.sh + PR_CSV=$(./scripts/test/detect_changed_detectors.sh --pr-csv || true) + MAIN_CSV=$(./scripts/test/detect_changed_detectors.sh --main-csv || true) + NEW_LIST=$(./scripts/test/detect_changed_detectors.sh --new-only || true) + NEW_CSV=$(echo "$NEW_LIST" | paste -sd, -) + echo "PR detectors: $PR_CSV" + echo "Main detectors: $MAIN_CSV" + echo "New detectors: $NEW_CSV" + echo "pr_csv=$PR_CSV" >> "$GITHUB_OUTPUT" + echo "main_csv=$MAIN_CSV" >> "$GITHUB_OUTPUT" + echo "new_csv=$NEW_CSV" >> "$GITHUB_OUTPUT" + if [[ -n "$PR_CSV" ]]; then + echo "any_changed=true" >> "$GITHUB_OUTPUT" + else + echo "any_changed=false" >> "$GITHUB_OUTPUT" + fi + + # Sticky comment: find any prior detector-bench comment on the PR by + # the marker substring and update it in place. The marker — kept in + # sync with STICKY_COMMENT_MARKER in scripts/test/diff_corpora_results.py — + # has to appear in BOTH the skip body and the diff body so the same + # comment flips between them as iterative pushes change which path + # fires. Skip body is only posted on pull_request events; workflow_dispatch + # runs with no changed detectors silently finish without posting. + - name: Find existing skip comment + if: steps.detect.outputs.any_changed != 'true' && github.event_name == 'pull_request' + id: find_skip_comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Post or update skip comment + if: steps.detect.outputs.any_changed != 'true' && github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5 + with: + comment-id: ${{ steps.find_skip_comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + edit-mode: replace + body: | + + ## Corpora Test Results + + No detector regex or keyword changes in this PR. Bench skipped. + + - name: Configure AWS credentials + if: steps.detect.outputs.any_changed == 'true' + uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + # Cache the main scan results by merge-base + scoped detector set. + # On subsequent pushes to the same PR without a rebase, both are + # identical, so the main scan (35 GB of S3 streaming + trufflehog) is + # skipped entirely. + - name: Restore main scan cache + id: main_scan_cache + if: steps.detect.outputs.any_changed == 'true' && steps.detect.outputs.main_csv != '' + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: /tmp/results-main.jsonl + key: main-scan-v1-${{ steps.merge_base.outputs.sha }}-${{ steps.detect.outputs.main_csv }} + + # Two independent builds run in parallel: + # A) prepare main worktree → build main binary (git I/O then CPU) + # Skipped on main scan cache hit or when main_csv is empty + # (all changed detectors are new — no baseline needed). + # B) build PR binary (CPU, no dependencies) + - name: Build binaries + if: steps.detect.outputs.any_changed == 'true' + shell: bash + env: + MERGE_BASE: ${{ steps.merge_base.outputs.sha }} + MAIN_CSV: ${{ steps.detect.outputs.main_csv }} + MAIN_SCAN_CACHE_HIT: ${{ steps.main_scan_cache.outputs.cache-hit }} + run: | + set -o pipefail + + # Chain A: prepare worktree, then build main binary. + # Skipped when main scan results are already cached, or when all + # changed detectors are new (main_csv empty — no baseline needed). + if [[ -n "$MAIN_CSV" && "$MAIN_SCAN_CACHE_HIT" != 'true' ]]; then + ( + git worktree add /tmp/trufflehog-main-src "$MERGE_BASE" + cd /tmp/trufflehog-main-src + CGO_ENABLED=0 go build -o /tmp/trufflehog-main . + ) & + PID_MAIN_BUILD=$! + fi + + # Chain B: build PR binary (no dependencies). + CGO_ENABLED=0 go build -o /tmp/trufflehog-pr . & + PID_PR_BUILD=$! + + [[ -n "${PID_MAIN_BUILD:-}" ]] && { wait $PID_MAIN_BUILD || { echo "Main binary build failed" >&2; exit 1; }; } + wait $PID_PR_BUILD || { echo "PR binary build failed" >&2; exit 1; } + + # PR and main scans share a single S3 stream per dataset file, teed to + # both binaries simultaneously. The main side is skipped on a cache hit + # (results already in /tmp/results-main.jsonl) or when main_csv is empty + # (PR adds only new detectors — no overlap with main). + - name: Run corpora tests + if: steps.detect.outputs.any_changed == 'true' + shell: bash + env: + PR_CSV: ${{ steps.detect.outputs.pr_csv }} + MAIN_CSV: ${{ steps.detect.outputs.main_csv }} + MAIN_SCAN_CACHE_HIT: ${{ steps.main_scan_cache.outputs.cache-hit }} + run: | + set -o pipefail + files=() + while IFS= read -r dataset; do + [[ -z "$dataset" ]] && continue + files+=("$dataset") + done <<< "$DATASETS" + + export TRUFFLEHOG_BIN=/tmp/trufflehog-pr + export OUTPUT_JSONL=/tmp/results-pr.jsonl + export INCLUDE_DETECTORS="$PR_CSV" + + if [[ -n "$MAIN_CSV" && "$MAIN_SCAN_CACHE_HIT" != 'true' ]]; then + # Dual-binary: single S3 download teed to both PR and main binaries. + export TRUFFLEHOG_BIN_MAIN=/tmp/trufflehog-main + export OUTPUT_JSONL_MAIN=/tmp/results-main.jsonl + export INCLUDE_DETECTORS_MAIN="$MAIN_CSV" + elif [[ -z "$MAIN_CSV" ]]; then + echo "No overlapping detectors in main; skipping main scan." + : > /tmp/results-main.jsonl + else + echo "Main scan cache hit; skipping main scan." + fi + + ./scripts/test/detector_corpora_test.sh "${files[@]}" \ + || { echo "Corpora scan failed" >&2; exit 1; } + + - name: Save main scan cache + if: steps.detect.outputs.any_changed == 'true' && steps.detect.outputs.main_csv != '' && steps.main_scan_cache.outputs.cache-hit != 'true' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: /tmp/results-main.jsonl + key: main-scan-v1-${{ steps.merge_base.outputs.sha }}-${{ steps.detect.outputs.main_csv }} + + - name: Diff results + if: steps.detect.outputs.any_changed == 'true' + shell: bash + env: + CHANGED: ${{ steps.detect.outputs.pr_csv }} + NEW_DETECTORS: ${{ steps.detect.outputs.new_csv }} + run: | + set -o pipefail + python3 scripts/test/diff_corpora_results.py \ + /tmp/results-main.jsonl /tmp/results-pr.jsonl \ + --changed-detectors="$CHANGED" \ + --new-detectors="$NEW_DETECTORS" \ + > /tmp/diff-report.md + cat /tmp/diff-report.md + + # workflow_dispatch runs don't carry an issue context, so resolve the + # PR number by branch lookup. pull_request events fall through to the + # event's issue number. Output feeds the find/update pair below. + - name: Resolve PR number + if: steps.detect.outputs.any_changed == 'true' + id: resolve_pr + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + let issue_number; + if (context.eventName === 'workflow_dispatch') { + const pulls = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${context.ref.replace('refs/heads/', '')}`, + state: 'open', + }); + if (pulls.data.length === 0) { + core.setFailed(`No open PR found for branch ${context.ref}`); + return; + } + issue_number = pulls.data[0].number; + } else { + issue_number = context.issue.number; + } + core.setOutput('issue_number', issue_number); + + - name: Find existing diff comment + if: steps.detect.outputs.any_changed == 'true' + id: find_diff_comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4 + with: + issue-number: ${{ steps.resolve_pr.outputs.issue_number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Post or update diff comment + if: steps.detect.outputs.any_changed == 'true' + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5 + with: + comment-id: ${{ steps.find_diff_comment.outputs.comment-id }} + issue-number: ${{ steps.resolve_pr.outputs.issue_number }} + edit-mode: replace + body-path: /tmp/diff-report.md diff --git a/.github/workflows/detector-tests.yml b/.github/workflows/detector-tests.yml index a777ccb6aa84..5e451573ce4e 100644 --- a/.github/workflows/detector-tests.yml +++ b/.github/workflows/detector-tests.yml @@ -14,13 +14,13 @@ jobs: contents: "read" id-token: "write" steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - name: Install gotestsum - uses: jaxxstorm/action-install-gh-release@v1.14.0 + uses: jaxxstorm/action-install-gh-release@25e24d2d23ae098373794ef1d6faecb48ee52da8 # v3.0.0 with: repo: gotestyourself/gotestsum - - uses: rwx-research/setup-captain@v1 + - uses: rwx-research/setup-captain@f761ee3671e4c027299f8a5842a53fba8ddd8e37 # v1.4.1 - name: Test Go run: | export CGO_ENABLED=1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 81183974787e..907869be5820 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,24 +15,24 @@ jobs: name: golangci-lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" - name: golangci-lint - uses: golangci/golangci-lint-action@v7 + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9 with: # NOTE: Version and args must match scripts/lint.sh - version: v2.11.4 - args: --disable errcheck,staticcheck --enable bodyclose,copyloopvar,misspell --timeout 10m + version: v2.13.2 + args: --enable bodyclose,copyloopvar,misspell --timeout 10m man-page-staleness: name: man-page-staleness runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" - name: Regenerate man page run: make man - name: Check for staleness @@ -48,7 +48,7 @@ jobs: image: returntocorp/semgrep if: (github.actor != 'dependabot[bot]') steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: semgrep --config=hack/semgrep-rules/detectors.yaml pkg/detectors/ checksecretparts: # Reports detector packages that construct detectors.Result without @@ -56,9 +56,14 @@ jobs: name: checksecretparts runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" + # hack/checksecretparts only needs the standard library, so this job + # finishes before any other and would otherwise claim the shared + # go.sum-keyed cache with an empty module cache. Cache keys are + # write-once, so that starves every later job on the branch. + cache: false - name: Run checksecretparts run: go run ./hack/checksecretparts -fail ./pkg/detectors diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index a75bbd98f20f..beb4711d91a2 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -11,15 +11,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 ref: ${{ github.head_ref }} - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" - name: Run Head run: | @@ -54,7 +54,7 @@ jobs: echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 ref: ${{ env.PREVIOUS_TAG }} diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml index 75db4b81ba83..3486dde5a7b1 100644 --- a/.github/workflows/release-bot.yml +++ b/.github/workflows/release-bot.yml @@ -14,12 +14,12 @@ jobs: steps: - name: Login to GCP id: auth - uses: "google-github-actions/auth@v2" + uses: "google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093" # v3 with: credentials_json: ${{ secrets.GCP_SA_TRUFFLE_RELEASE_BOT }} - name: Login to GAR - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: us-central1-docker.pkg.dev username: _json_key diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e12dbb0ca5e2..1500ff0354a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,28 +18,28 @@ jobs: steps: # Setup steps - no external side effects. - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Docker Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Docker Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" - name: Cosign install - uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2 + uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 - name: Install UPX run: | sudo apt-get update @@ -64,7 +64,7 @@ jobs: # version. The release is NOT marked latest (make_latest: false), so # /releases/latest still points to the previous good release. - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v6 + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7 with: distribution: goreleaser-pro version: latest diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml index 0b0614a99cd1..e0dd2d72c12f 100644 --- a/.github/workflows/secrets.yml +++ b/.github/workflows/secrets.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 ref: ${{ github.head_ref }} diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 8390c5d9d1a6..2e483605c7a7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -3,37 +3,49 @@ name: Smoke on: pull_request: +permissions: + contents: read + jobs: smoke: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" + - name: Build + run: go build -o "$RUNNER_TEMP/trufflehog" . - name: Smoke + timeout-minutes: 10 run: | set -e - go run . git https://github.com/dustin-decker/secretsandstuff.git > /dev/null - go run . github --repo https://github.com/dustin-decker/secretsandstuff.git > /dev/null + "$RUNNER_TEMP/trufflehog" git https://github.com/dustin-decker/secretsandstuff.git > /dev/null + "$RUNNER_TEMP/trufflehog" github --repo https://github.com/dustin-decker/secretsandstuff.git > /dev/null zombies: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 15 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" + - name: Build + run: go build -o "$RUNNER_TEMP/trufflehog" . + # The timeout covers only the scans, not the build, so a cold Go cache + # cannot eat into the budget that detects the #3379 deadlock. - name: Run trufflehog + timeout-minutes: 5 run: | set -e - go run . git --no-verification file://. > /dev/null + "$RUNNER_TEMP/trufflehog" git --no-verification file://. > /dev/null # This case previously had a deadlock issue and left zombies after trufflehog exited #3379 - go run . git --no-verification https://github.com/git-test-fixtures/binary.git > /dev/null + "$RUNNER_TEMP/trufflehog" git --no-verification https://github.com/git-test-fixtures/binary.git > /dev/null - name: Check for running git processes and zombies run: | if pgrep -x "git" > /dev/null diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc98c2327855..1d756d4adaeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,13 +18,13 @@ jobs: id-token: "write" steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" - id: "auth" - uses: "google-github-actions/auth@v2" + uses: "google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093" # v3 with: workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider" service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com" @@ -38,7 +38,7 @@ jobs: if: ${{ success() || failure() }} # always run this step, even if there were previous errors - name: Upload test results to BuildPulse for flaky test detection if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled. - uses: buildpulse/buildpulse-action@main + uses: buildpulse/buildpulse-action@d4d8e00c645a2e3db0419a43664bbcf868080234 # v0.12.0 with: account: 79229934 repository: 77726177 @@ -48,7 +48,7 @@ jobs: secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} tags: integration - name: Annotate test results - uses: mikepenz/action-junit-report@v5 + uses: mikepenz/action-junit-report@a9170d5795813c01ab4901ffb045b52bab4ab09d # v6 if: success() || failure() # always run even if the previous step fails with: report_paths: "tmp/test-results/*.xml" @@ -60,10 +60,10 @@ jobs: contents: "read" steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: - go-version: "1.25" + go-version: "1.27" - name: Test run: make test-community diff --git a/.gitignore b/.gitignore index 6abf4e766574..e78f7fe9acf0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ dist .env *.test +[._]*.sw[a-p] + +# rapid property test failure recordings, for local reproduction only +testdata/rapid/ # binary trufflehog @@ -10,3 +14,7 @@ tmp/go-test.json .captain/detectors/quarantines.yaml .captain/detectors/flakes.yaml .vscode + +# Python +__pycache__/ +*.pyc diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000000..1a62389565b2 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,11 @@ +version: "2" +linters: + enable: + - depguard + settings: + depguard: + rules: + no-stdlib-regexp: + deny: + - pkg: "regexp$" + desc: "use wasilibs/go-re2 unless necessary" diff --git a/CODEOWNERS b/CODEOWNERS index 501fe8997594..236070ae1559 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,56 +1,17 @@ -# catch-all -* @trufflesecurity/product-eng +# Default -- Integrations owns OSS, community, detectors, sources, analyzers +* @trufflesecurity/integrations -# Shared -pkg/decoders/ @trufflesecurity/Scanning @trufflesecurity/OSS -pkg/engine/ @trufflesecurity/Scanning @trufflesecurity/OSS -pkg/gitparse/ @trufflesecurity/Scanning @trufflesecurity/OSS -pkg/giturl/ @trufflesecurity/Scanning @trufflesecurity/OSS -pkg/handlers/ @trufflesecurity/Scanning @trufflesecurity/OSS -pkg/iobuf/ @trufflesecurity/Scanning @trufflesecurity/OSS -pkg/sanitizer/ @trufflesecurity/Scanning @trufflesecurity/OSS -proto/ @trufflesecurity/Scanning @trufflesecurity/Integrations +# Scan engine and pipeline +pkg/decoders/ @trufflesecurity/scanning +pkg/engine/ @trufflesecurity/scanning +pkg/handlers/ @trufflesecurity/scanning +pkg/iobuf/ @trufflesecurity/scanning +pkg/sanitizer/ @trufflesecurity/scanning +pkg/writers/ @trufflesecurity/scanning -# Scanning -pkg/sources/ @trufflesecurity/Scanning -pkg/writers/ @trufflesecurity/Scanning +# Contracts -- co-owned, changes affect both teams +proto/ @trufflesecurity/integrations @trufflesecurity/scanning -# Integrations -pkg/sources/circleci/ @trufflesecurity/Integrations -pkg/sources/docker/ @trufflesecurity/Integrations -pkg/sources/elasticsearch/ @trufflesecurity/Integrations -pkg/sources/filesystem/ @trufflesecurity/Integrations -pkg/sources/gcs/ @trufflesecurity/Integrations -pkg/sources/git/ @trufflesecurity/Integrations -pkg/sources/github/ @trufflesecurity/Integrations -pkg/sources/gitlab/ @trufflesecurity/Integrations -pkg/sources/jenkins/ @trufflesecurity/Integrations -pkg/sources/postman/ @trufflesecurity/Integrations -pkg/sources/s3/ @trufflesecurity/Integrations -pkg/sources/travisci/ @trufflesecurity/Integrations -proto/detector_type.proto @trufflesecurity/Integrations - -# OSS -pkg/detectors/ @trufflesecurity/OSS -pkg/common/ @trufflesecurity/OSS -pkg/custom_detectors/ @trufflesecurity/OSS -pkg/analyzer/ @trufflesecurity/OSS -pkg/engine/defaults/defaults.go @trufflesecurity/OSS -pkg/engine/defaults/defaults_test.go @trufflesecurity/OSS - -# critical detectors -pkg/detectors/aws/ @trufflesecurity/backend -pkg/detectors/gcp/ @trufflesecurity/backend -pkg/detectors/azure/ @trufflesecurity/backend -pkg/detectors/okta/ @trufflesecurity/backend -pkg/detectors/privatekey/ @trufflesecurity/backend -pkg/detectors/slack/ @trufflesecurity/backend -pkg/detectors/slackwebhook/ @trufflesecurity/backend -pkg/detectors/microsoftteamswebhook/ @trufflesecurity/backend -pkg/detectors/twilio/ @trufflesecurity/backend -pkg/detectors/sendgrid/ @trufflesecurity/backend -pkg/detectors/gitlab/ @trufflesecurity/backend -pkg/detectors/gitlabv2/ @trufflesecurity/backend -pkg/detectors/github/ @trufflesecurity/backend -pkg/detectors/github_old/ @trufflesecurity/backend -pkg/detectors/githubapp/ @trufflesecurity/backend +# Detectors are fully owned by integrations +proto/detector_type.proto @trufflesecurity/integrations +pkg/engine/defaults/ @trufflesecurity/integrations diff --git a/Makefile b/Makefile index e9065d9ed86e..bbf06b773ad5 100644 --- a/Makefile +++ b/Makefile @@ -1,70 +1,86 @@ -PROTOS_IMAGE ?= trufflesecurity/protos:1.22 +.DEFAULT_GOAL := help +.PHONY: help +.PHONY: run +.PHONY: run-debug +.PHONY: dogfood +.PHONY: install .PHONY: check .PHONY: lint +.PHONY: test-failing .PHONY: test +.PHONY: test-integration .PHONY: test-race -.PHONY: run -.PHONY: install +.PHONY: test-detectors +.PHONY: test-community +.PHONY: bench .PHONY: protos .PHONY: protos-windows -.PHONY: vendor -.PHONY: dogfood +.PHONY: release-protos-image .PHONY: man +.PHONY: test-release + +PROTOS_IMAGE ?= trufflesecurity/protos:1.22 -dogfood: +help: ## Display this help message + @echo "Usage: make [target]" + @echo + @echo "Targets:" + @awk '/^[a-zA-Z0-9_-]+:.*## / {printf "%s : %s\n", $$1, substr($$0, index($$0, "##") + 3)}' $(MAKEFILE_LIST) | sort | column -t -s ':' + @echo + +run: ## Run a git scan on this repository + CGO_ENABLED=0 go run . git file://. --json + +run-debug: ## Run a git scan on this repository with enhanced logging (level 2) CGO_ENABLED=0 go run . git file://. --json --log-level=2 -install: +dogfood: run-debug + +install: ## Run go install CGO_ENABLED=0 go install . -check: +check: ## Run go fmt and go vet go fmt $(shell go list ./... | grep -v /vendor/) go vet $(shell go list ./... | grep -v /vendor/) -lint: +lint: ## Run the linter ./scripts/lint.sh -test-failing: +test-failing: ## Run tests, displaying only the failures CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) | grep FAIL -test: +test: ## Run tests CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) -test-integration: +test-integration: ## Run integration tests CGO_ENABLED=0 go test -timeout=5m -tags=integration $(shell go list ./... | grep -v /vendor/) -test-race: +test-race: ## Run the go race condition checker CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/) -test-detectors: +test-detectors: ## Run detector tests CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors) -test-community: +test-community: ## Run a subset of tests intended for use by the OSS community CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/sources | grep -v pkg/analyzer/analyzers) -bench: +bench: ## Run benchmarks CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench . -run: - CGO_ENABLED=0 go run . git file://. --json - -run-debug: - CGO_ENABLED=0 go run . git file://. --json --log-level=2 - -protos: +protos: ## Regenerate protocol buffer code docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh" -protos-windows: +protos-windows: ## Regenerate protocol buffer code on Windows docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh" -release-protos-image: +release-protos-image: ## Build a Docker image suitable for generating the protocol buffer code docker buildx build --push --platform=linux/amd64,linux/arm64 \ -t ${PROTOS_IMAGE} -f hack/Dockerfile.protos . -man: +man: ## Build man pages @mkdir -p docs/man CGO_ENABLED=0 go run . --generate-man-page > docs/man/trufflehog.1 -test-release: +test-release: ## Test a goreleaser release goreleaser release --clean --skip=publish,sign --snapshot diff --git a/README.md b/README.md index 20f309afec1e..a19abac8863a 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,13 @@ Timestamp: 2022-06-16 10:17:40 -0700 PDT trufflehog github --org=trufflesecurity --results=verified ``` -## 3: Scan a GitHub Repo for only verified secrets and get JSON output +## 3: Scan a GitHub Org excluding archived repositories + +```bash +trufflehog github --org=trufflesecurity --exclude-archived +``` + +## 4: Scan a GitHub Repo for only verified secrets and get JSON output Command: @@ -227,37 +233,39 @@ Expected output: ... ``` -## 4: Scan a GitHub Repo + its Issues and Pull Requests +TruffleHog can also output [SARIF](https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html) with `--sarif` instead of `--json`. GitHub code scanning understands SARIF natively, so uploading it surfaces findings inline on pull request diffs and in the repository's Security tab, and tracks findings as new/fixed across scans instead of reporting the same one every run — see the [TruffleHog Github Action](#octocat-trufflehog-github-action) section below for how to upload it. Note that, unlike the other output formats, SARIF results are buffered in memory for the full scan and written out at the end, since SARIF requires a single JSON document rather than a stream — fine for typical scans, but scans producing a very large number of results will use proportionally more memory. + +## 5: Scan a GitHub Repo + its Issues and Pull Requests ```bash trufflehog github --repo=https://github.com/trufflesecurity/test_keys --issue-comments --pr-comments ``` -## 5: Scan an S3 bucket for high-confidence results (verified + unknown) +## 6: Scan an S3 bucket for high-confidence results (verified + unknown) ```bash trufflehog s3 --bucket= --results=verified,unknown ``` -## 6: Scan S3 buckets using IAM Roles +## 7: Scan S3 buckets using IAM Roles ```bash trufflehog s3 --role-arn= ``` -## 7: Scan a Github Repo using SSH authentication in Docker +## 8: Scan a Github Repo using SSH authentication in Docker ```bash docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys ``` -## 8: Scan individual files or directories +## 9: Scan individual files or directories ```bash trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir ``` -## 9: Scan a local git repo +## 10: Scan a local git repo Clone the git repo. For example [test keys](git@github.com:trufflesecurity/test_keys.git) repo. ```bash @@ -271,13 +279,13 @@ trufflehog git file://test_keys --results=verified,unknown To guard against malicious git configs in local scanning (see CVE-2025-41390), TruffleHog clones local git repositories to a temporary directory prior to scanning. This follows [Git's security best practices](https://git-scm.com/docs/git#_security). If you want to specify a custom path to clone the repository to (instead of tmp), you can use the `--clone-path` flag. If you'd like to skip the local cloning process and scan the repository directly (only do this for trusted repos), you can use the `--trust-local-git-config` flag. -## 10: Scan GCS buckets for only verified secrets +## 11: Scan GCS buckets for only verified secrets ```bash trufflehog gcs --project-id= --cloud-environment --results=verified ``` -## 11: Scan a Docker image for only verified secrets +## 12: Scan a Docker image for only verified secrets Use the `--image` flag multiple times to scan multiple images. @@ -292,7 +300,7 @@ trufflehog docker --image docker://new_image:tag --results=verified trufflehog docker --image file://path_to_image.tar --results=verified ``` -## 12: Scan in CI +## 13: Scan in CI Set the `--since-commit` flag to your default branch that people merge into (ex: "main"). Set the `--branch` flag to your PR's branch name (ex: "feature-1"). Depending on the CI/CD platform you use, this value can be pulled in dynamically (ex: [CIRCLE_BRANCH in Circle CI](https://circleci.com/docs/variables/) and [TRAVIS_PULL_REQUEST_BRANCH in Travis CI](https://docs.travis-ci.com/user/environment-variables/)). If the repo is cloned and the target branch is already checked out during the CI/CD workflow, then `--branch HEAD` should be sufficient. The `--fail` flag will return an 183 error code if valid credentials are found. @@ -300,7 +308,7 @@ Set the `--since-commit` flag to your default branch that people merge into (ex: trufflehog git file://. --since-commit main --branch feature-1 --results=verified,unknown --fail ``` -## 13: Scan a Postman workspace +## 14: Scan a Postman workspace Use the `--workspace-id`, `--collection-id`, `--environment` flags multiple times to scan multiple targets. @@ -308,13 +316,13 @@ Use the `--workspace-id`, `--collection-id`, `--environment` flags multiple time trufflehog postman --token= --workspace-id= ``` -## 14: Scan a Jenkins server +## 15: Scan a Jenkins server ```bash trufflehog jenkins --url https://jenkins.example.com --username admin --password admin ``` -## 15: Scan an Elasticsearch server +## 16: Scan an Elasticsearch server ### Scan a Local Cluster @@ -342,7 +350,7 @@ trufflehog elasticsearch \ --api-key 'MlVtVjBZ...ZSYlduYnF1djh3NG5FQQ==' ``` -## 16. Scan a GitHub Repository for Cross Fork Object References and Deleted Commits +## 17. Scan a GitHub Repository for Cross Fork Object References and Deleted Commits The following command will enumerate deleted and hidden commits on a GitHub repository and then scan them for secrets. This is an alpha release feature. @@ -356,21 +364,25 @@ In addition to the normal TruffleHog output, the `--object-discovery` flag creat For more information on Cross Fork Object References, please [read our blog post](https://trufflesecurity.com/blog/anyone-can-access-deleted-and-private-repo-data-github). -## 17. Scan Hugging Face +## 18. Scan Hugging Face -### Scan a Hugging Face Model, Dataset or Space +### Scan a Hugging Face Model, Dataset, Space or Bucket ```bash -trufflehog huggingface --model --space --dataset +trufflehog huggingface \ + --model \ + --dataset \ + --space \ + --bucket ``` -### Scan all Models, Datasets and Spaces belonging to a Hugging Face Organization or User +### Scan all Models, Datasets, Spaces and Buckets belonging to a Hugging Face Organization or User ```bash trufflehog huggingface --org --user ``` -(Optionally) When scanning an organization or user, you can skip an entire class of resources with `--skip-models`, `--skip-datasets`, `--skip-spaces` OR a particular resource with `--ignore-models `, `--ignore-datasets `, `--ignore-spaces `. +(Optionally) When scanning an organization or user, you can skip an entire class of resources with `--skip-all-models`, `--skip-all-datasets`, `--skip-all-spaces`, `--skip-all-buckets` OR a particular resource with `--ignore-models `, `--ignore-datasets `, `--ignore-spaces `, `--ignore-buckets `. ### Scan Discussion and PR Comments @@ -378,7 +390,7 @@ trufflehog huggingface --org --user trufflehog huggingface --model --include-discussions --include-prs ``` -## 18. Scan stdin Input +## 19. Scan stdin Input ```bash aws s3 cp s3://example/gzipped/data.gz - | gunzip -c | trufflehog stdin @@ -394,6 +406,8 @@ aws s3 cp s3://example/gzipped/data.gz - | gunzip -c | trufflehog stdin - A verified result means TruffleHog confirmed the credential is valid by testing it against the service's API. For private keys, we've confirmed the key can be used live for SSH or SSL authentication. Check out our Driftwood blog post to learn more [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/) - Is there an easy way to ignore specific secrets? - If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets. +- Can I find secrets with `trufflehog:ignore` included? + - Pass `--no-ignore-tag` to report results even when their line carries a `trufflehog:ignore` comment. This is useful for reviewing previously accepted findings. # :newspaper: What's new in v3? @@ -413,7 +427,7 @@ For every potential credential that is detected, we've painstakingly implemented - **unverified**: Credential detected but not confirmed valid (may be invalid, expired, or verification disabled) - **unknown**: Verification attempted but failed due to errors, such as a network or API failure -For example, the [AWS credential detector](pkg/detectors/aws/aws.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active. +For example, the [AWS credential detector](pkg/detectors/aws/access_keys/accesskey.go) performs a `GetCallerIdentity` API call against the AWS API to verify if an AWS credential is active. # :memo: Usage @@ -422,6 +436,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan: - git - github - gitlab +- huggingface - docker - s3 - filesystem (files and directories) @@ -453,6 +468,8 @@ Flags: --[no-]json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources. --[no-]github-actions Output in GitHub Actions format. + --[no-]sarif Output in SARIF format for upload to GitHub code scanning (e.g. + via github/codeql-action/upload-sarif). --concurrency=12 Number of concurrent workers. --[no-]no-verification Don't verify the results. --results=RESULTS Specifies which type(s) of results to output: verified (confirmed @@ -582,7 +599,7 @@ The regex detectors can be used with any subcommand, while the sources defined in configuration are only for the `multi-scan` subcommand. The configuration format for sources can be found on Truffle Security's -[source configuration documentation page](https://docs.trufflesecurity.com/scan-data-for-secrets). +[source configuration documentation page](https://trufflesecurity.com/docs/connect-sources). Example GitHub source configuration and [options reference](https://docs.trufflesecurity.com/github#Fvm1I): @@ -625,6 +642,58 @@ Multiple roles can be passed as separate arguments. The following command will a trufflehog s3 --role-arn= --role-arn= ``` +### Narrowing a scan to specific objects + +Large buckets often hold data that is never worth scanning, such as archives, media, and build artifacts. Two sets of flags keep those objects out of a scan. Both are matched against the object key before the object is downloaded, so a skipped object costs no GET request. + +Scan only the objects under one or more key prefixes: + +```bash +trufflehog s3 --bucket= --include-prefix=infra/ --include-prefix=services/ +``` + +Skip the objects under a key prefix: + +```bash +trufflehog s3 --bucket= --exclude-prefix=projects/archived/ +``` + +Both prefix flags can be used together to scan a subtree while leaving one part of it out. An object that matches an exclude prefix is always skipped, even if it also matches an include prefix: + +```bash +trufflehog s3 --bucket= --include-prefix=src/ --exclude-prefix=src/vendor/ +``` + +Prefixes are matched literally, not as globs, and they are not confined to a path boundary. `--include-prefix=log` matches both `logs/app.txt` and `logs-archive/app.txt`. + +File extensions are filtered separately. Write them without a leading dot: + +```bash +trufflehog s3 --bucket= --exclude-extension=zip --exclude-extension=mp4 +``` + +```bash +trufflehog s3 --bucket= --include-extension=tf --include-extension=yaml +``` + +Unlike the prefix flags, `--include-extension` and `--exclude-extension` cannot be combined, because naming the extensions to scan already excludes every other one. Using both fails at startup. + +Extension matching is case insensitive, so `--exclude-extension=zip` also skips `BACKUP.ZIP`. An object whose key has no extension at all, such as `Makefile`, matches no entry: it is skipped when `--include-extension` is set, and kept when only `--exclude-extension` is set. + +Two things to watch when using `--include-extension`. A dotfile counts as all extension, so `.env` has the extension `env` and any include list that leaves out `env` will skip every `.env` file in the bucket. Since `.env` files are a common place for secrets to sit, add `--include-extension=env` unless you mean to skip them. And only the last extension counts, so `backup.tar.gz` has the extension `gz`: `--exclude-extension=tar.gz` matches nothing, while `--exclude-extension=gz` works. + +These flags also apply to object keys only, not to files inside an archive. `--exclude-extension=mp4` skips `clip.mp4` sitting in the bucket, but not a `clip.mp4` packed inside `media.zip`. Excluding `zip` skips the archive entirely. + +TruffleHog asks S3 to list only the keys under the include prefixes, so a prefix that covers a small part of a large bucket is listed quickly. Exclude prefixes and extensions are applied after listing, so they cut the cost of downloading objects but not the cost of listing them. + +Prefixes and extensions are applied together. An object has to pass both to be scanned, so the command below scans `src/main.tf` but skips both `src/bundle.zip` and `docs/guide.tf`: + +```bash +trufflehog s3 --bucket= --include-prefix=src/ --exclude-extension=zip +``` + +These flags narrow the objects within a bucket. To narrow which buckets are scanned, use `--bucket` or `--ignore-bucket`, which unlike the prefix flags cannot be used together. + Exit Codes: - 0: No errors and no results were found. @@ -706,10 +775,25 @@ TruffleHog statically detects [https://canarytokens.org/](https://canarytokens.o head: # optional # Extra args to be passed to the trufflehog cli. extra_args: --log-level=2 --results=verified,unknown + # Scan with a specific TruffleHog version (default: latest). + version: + # Docker image to pull. Override to use a registry mirror (default: ghcr.io/trufflesecurity/trufflehog). + image: ``` If you'd like to specify specific `base` and `head` refs, you can use the `base` argument (`--since-commit` flag in TruffleHog CLI) and the `head` argument (`--branch` flag in the TruffleHog CLI). We only recommend using these arguments for very specific use cases, where the default behavior does not work. +To upload results to GitHub code scanning instead, run TruffleHog directly with `--sarif` and pass the output to `github/codeql-action/upload-sarif`: + +```yaml +- name: TruffleHog + run: trufflehog filesystem . --sarif --no-verification > results.sarif +- name: Upload SARIF results + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif +``` + #### Advanced Usage: Scan entire branch ``` @@ -768,6 +852,8 @@ Custom Detectors support a few different filtering mechanisms: entropy, regex ta and excluded word lists checked against the secret (captured group if present, entire match if capture group is not present). Note that if your custom detector has multiple `regex` set (in this example `hogID`, and `hogToken`), then the filters get applied to each regex. [Here](examples/generic_with_filters.yml) is an example of a custom detector using these filters. +The `verify` section is optional — if you omit it, matches are still reported as unverified, no webhook required. This makes Custom Detectors useful for flagging generic hardcoded secrets (e.g. `*.password=`, `*.secret=`) in config files like `.properties`, `.env`, or `.yaml` that TruffleHog's built-in, verified detectors won't otherwise catch. [Here](examples/generic_config_secrets.yml) is an example tuned for that use case. + **NB:** This feature is alpha and subject to change. ### Regex Detector Example diff --git a/SECURITY.md b/SECURITY.md index eb52c5f85810..425205314811 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,22 +1,68 @@ -Please report security issues to security@trufflesec.com and include `trufflehog` in the subject line. If your vulnerability involves SSRF or outbound requests, please see our policy for that specific class of vulnerability below. +Please report security issues to security@trufflesec.com and include +`trufflehog` in the subject line. If your vulnerability involves SSRF or +outbound requests, please see our policy for that specific class of +vulnerability below. + ## Blind SSRF & Outbound Request Policy -Truffle Security treats blind SSRF (the ability to induce outbound requests without data retrieval) as a hardening opportunity rather than a vulnerability. We do not issue CVEs or formal advisories for reports showing outbound interactions unless they demonstrate a tangible security risk to users. -#### Policy Criteria -**Vulnerability (CVE Issued):** We will issue a CVE if a researcher demonstrates a clear exploit chain. For example: -- Credential Exfiltration: Forcing TruffleHog to send third-party secrets (discovered during a scan) or the host's own environment credentials (e.g., IAM metadata) to an attacker-controlled endpoint. -- Internal Exploitation: Using a blind request to trigger secondary vulnerabilities (e.g. RCE) on restricted internal services configured for defense-in-depth. +As part of its intended operation, TruffleHog makes network requests to +upstream secret providers in order to verify liveness of each secret. + +Truffle Security treats blind SSRF (i.e., the ability to induce outbound +requests without data retrieval) as a hardening opportunity rather than +a vulnerability. + +We do not issue CVEs or formal advisories for reports showing outbound +interactions unless they demonstrate a tangible security risk to users. + +### Policy Criteria + +#### Vulnerability (CVE Issued): +We will issue a CVE if a researcher demonstrates a clear exploit chain. +For example: + +- Credential Exfiltration: Forcing TruffleHog to send third-party + secrets (discovered during a scan) or the host's own environment + credentials (e.g., IAM metadata) to an attacker-controlled endpoint. + +- Internal Exploitation: Using a blind request to trigger secondary + vulnerabilities (e.g. RCE) on restricted internal services configured + for defense-in-depth. + +#### Hardening Opportunity (No CVE): +We will not issue a CVE for: -**Hardening (No CVE):** We generally will not issue a CVE for: -- Reflected Payloads: Inducing a request to an attacker-controlled URL that was already present in the scanned source code (i.e., the attacker receiving their own data back). -- Basic Outbound Control: Demonstrating control over the request URL, Path, or Body, without demonstrating a path to credential leakage or internal system exploitation. -- Service Probing: Simple open/closed port verification or basic interaction with internal services (e.g., triggering a GET request to a local web server) without a demonstrated compromise of data or system integrity. -- Secondary Vulnerability Dependencies: Where the impact relies entirely on the pre-existing lack of authentication, misconfiguration, or known vulnerabilities of a third-party internal service. +- **Reflected Payloads:** Inducing a request to an attacker-controlled endpoint + that requires the attacker to have write access to the scan input (i.e., the + attacker receiving data back that they already had access to). + +- **Basic Outbound Control:** Demonstrating control over the request + URL, Path, or Body, without demonstrating a path to credential leakage + or internal system exploitation. + +- **Service Probing:** Simple open/closed port verification or basic + interaction with internal services (e.g., triggering a GET request to + a local web server) without a demonstrated compromise of data or + system integrity. + +- **Secondary Vulnerability Dependencies:** Where the impact relies + entirely on the pre-existing lack of authentication, misconfiguration, + or known vulnerabilities of a third-party internal service. ### Submission Guidelines + To help us evaluate your report, please specify: -- Level of Control: Which request components are controllable (Method, Host, Path, Headers, or Body)? -- Secret Context: Can you prove that a legitimate secret (not the attacker's payload) is attached to or contained within the outbound request? -- Target Reach: Can the request reach restricted internal IPs (e.g., 127.0.0.1 or 169.254.169.254)? -- Demonstrated Impact: What is the specific risk to a user or environment beyond a simple DNS/HTTP interaction? + +- **Level of Control:** Which request components are controllable (Method, + Host, Path, Headers, or Body)? + +- **Secret Context:** Can you prove that a legitimate secret that the attacker + did not already have access to (i.e., not the attacker's payload) is attached + to or contained within the outbound request? + +- **Target Reach:** Can the request reach restricted internal IPs (e.g., + 127.0.0.1 or 169.254.169.254)? + +- **Demonstrated Impact:** What is the specific risk to a user or environment + beyond a simple DNS/HTTP interaction? diff --git a/action.yml b/action.yml index 2acb0ad23f81..1e2af3fbb9ef 100644 --- a/action.yml +++ b/action.yml @@ -22,6 +22,10 @@ inputs: default: "latest" description: Scan with this trufflehog cli version. required: false + image: + default: "ghcr.io/trufflesecurity/trufflehog" + description: Docker image to use. Override to point at a registry mirror. + required: false branding: icon: "shield" color: "green" @@ -37,6 +41,7 @@ runs: ARGS: ${{ inputs.extra_args }} COMMIT_IDS: ${{ toJson(github.event.commits.*.id) }} VERSION: ${{ inputs.version }} + IMAGE: ${{ inputs.image }} run: | ########################################## ## ADVANCED USAGE ## @@ -94,7 +99,7 @@ runs: ## Run TruffleHog ## ########################################## docker run --rm -v .:/tmp -w /tmp \ - ghcr.io/trufflesecurity/trufflehog:${VERSION} \ + "${IMAGE}:${VERSION}" \ git file:///tmp/ \ --since-commit \ ${BASE:-''} \ diff --git a/docs/man/trufflehog.1 b/docs/man/trufflehog.1 index ab4f99a7ddd8..49d769f50253 100644 --- a/docs/man/trufflehog.1 +++ b/docs/man/trufflehog.1 @@ -38,6 +38,9 @@ Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources. \fB--github-actions\fR Output in GitHub Actions format. .TP +\fB--sarif\fR +Output in SARIF format for upload to GitHub code scanning (e.g. via github/codeql-action/upload-sarif). +.TP \fB--concurrency=N\fR Number of concurrent workers. .TP @@ -62,6 +65,9 @@ Only output first unverified result per chunk per detector if there are more tha \fB--filter-entropy=FILTER-ENTROPY\fR Filter unverified results with Shannon entropy. Start with 3.0. .TP +\fB--no-ignore-tag\fR +Report results even if the line has a 'trufflehog:ignore' comment. +.TP \fB--max-decode-depth=5\fR Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16). .TP @@ -119,6 +125,9 @@ Skip additional references. \fB--user-agent-suffix=USER-AGENT-SUFFIX\fR Suffix to add to User-Agent. .TP +\fB--drop-unverified-jwt-results\fR +Drop unverified results without any verification errors from the JWT detector. +.TP \fB--version\fR Show application version. .SH COMMANDS @@ -154,7 +163,7 @@ Scan bare repository (e.g. useful while using in pre-receive hooks) Custom path where the repository should be cloned (default: temp dir). .TP \fB--no-cleanup\fR -Do not delete cloned repositories after scanning (can only be used with --clone-path). +Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run. .TP \fB--trust-local-git-config\fR Trust local git config. @@ -223,10 +232,13 @@ Embed authentication credentials in repository URLs instead of using secure HTTP Custom path where the repository should be cloned (default: temp dir). .TP \fB--no-cleanup\fR -Do not delete cloned repositories after scanning (can only be used with --clone-path). +Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run. .TP \fB--ignore-gists\fR Ignore all gists in scan. +.TP +\fB--exclude-archived\fR +Exclude archived repositories from scan. .SS \fBgithub-experimental --repo=REPO []\fR Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery. @@ -277,10 +289,10 @@ Repositories to exclude in an org scan. This can also be a glob pattern. You can Embed authentication credentials in repository URLs instead of using secure HTTP headers .TP \fB--clone-path=CLONE-PATH\fR -Custom path where the repository should be cloned (default: temp dir) +Custom path where the repository should be cloned (default: temp dir). .TP \fB--no-cleanup\fR -Do not delete cloned repositories after scanning (can only be used with --clone-path). +Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run. .SS \fBfilesystem [] [...]\fR Find credentials in a filesystem. @@ -323,6 +335,24 @@ Name of S3 bucket to ignore. You can repeat this flag. Incompatible with --bucke .TP \fB--max-object-size=250MB\fR Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB) +.TP +\fB--endpoint=ENDPOINT\fR +Endpoint of an S3-compatible service to scan instead of AWS S3. (eg. https://s3.internal.example.com) +.TP +\fB--region=REGION\fR +Region used to sign requests. Defaults to us-east-1. +.TP +\fB--include-prefix=INCLUDE-PREFIX\fR +Only scan objects whose key starts with this prefix. You can repeat this flag. +.TP +\fB--exclude-prefix=EXCLUDE-PREFIX\fR +Skip objects whose key starts with this prefix. You can repeat this flag. Takes precedence over --include-prefix. +.TP +\fB--include-extension=INCLUDE-EXTENSION\fR +Only scan objects with this file extension, written without a leading dot (eg. tf). You can repeat this flag. Incompatible with --exclude-extension. +.TP +\fB--exclude-extension=EXCLUDE-EXTENSION\fR +Skip objects with this file extension, written without a leading dot (eg. zip). You can repeat this flag. Incompatible with --include-extension. .SS \fBgcs []\fR Find credentials in GCS buckets. @@ -490,7 +520,7 @@ Jenkins password Skip TLS verification .SS \fBhuggingface []\fR -Find credentials in HuggingFace datasets, models and spaces. +Find credentials in HuggingFace datasets, models, spaces and buckets. .TP \fB--endpoint="https://huggingface.co"\fR HuggingFace endpoint. @@ -504,6 +534,9 @@ HuggingFace space to scan. You can repeat this flag. Example: 'username/space' \fB--dataset=DATASET\fR HuggingFace dataset to scan. You can repeat this flag. Example: 'username/dataset' .TP +\fB--bucket=BUCKET\fR +HuggingFace bucket to scan. You can repeat this flag. Example: 'username/bucket' +.TP \fB--org=ORG\fR HuggingFace organization to scan. You can repeat this flag. Example: "trufflesecurity" .TP @@ -531,6 +564,12 @@ Spaces to ignore in scan. You can repeat this flag. Must use HuggingFace space f \fB--ignore-datasets=IGNORE-DATASETS\fR Datasets to ignore in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org) .TP +\fB--include-buckets=INCLUDE-BUCKETS\fR +Buckets to include in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org) +.TP +\fB--ignore-buckets=IGNORE-BUCKETS\fR +Buckets to ignore in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org) +.TP \fB--skip-all-models\fR Skip all model scans. (Only used with --user or --org) .TP @@ -540,6 +579,9 @@ Skip all space scans. (Only used with --user or --org) \fB--skip-all-datasets\fR Skip all dataset scans. (Only used with --user or --org) .TP +\fB--skip-all-buckets\fR +Skip all bucket scans. (Only used with --user or --org) +.TP \fB--include-discussions\fR Include discussions in scan. .TP diff --git a/examples/generic_config_secrets.yml b/examples/generic_config_secrets.yml new file mode 100644 index 000000000000..06a814c4c78f --- /dev/null +++ b/examples/generic_config_secrets.yml @@ -0,0 +1,47 @@ +detectors: +- name: generic-config-secret + keywords: + - secret + - password + - passwd + - pwd + - apikey + - api_key + - api-key + - token + - credential + - cred + - auth + regex: + secret: |- + (?i)[\w.-]{0,50}?(?:secret|passw(?:or)?d|pwd|api[_-]?key|token|credentials?|creds?|auth)[\w.-]{0,20}\s*[=:]\s*["'`]?([^\s"'`,;]{6,150})["'`]?\s*(?:$|[\r\n#;]) + entropy: 2.5 + # Applied to the captured value only (not the key), so a key path like + # `env.password` or `app.env.secret` doesn't cause a real hardcoded + # secret to be excluded just because "env" appears in the key. + exclude_regexes_capture: + - '(?i)^\$\{[^}]*\}$' + - '(?i)^\{\{[^}]*\}\}$' + - '(?i)^%[A-Z_]+%$' + - '(?i)^\$[A-Z_][A-Z0-9_]*$' + - '(?i)^(?:process\.env\.|os\.environ|getenv\(|System\.getenv\()' + - '(?i)^(?:vault|kms|secretsmanager|parameterstore)://' + exclude_words: + - "changeme" + - "change_me" + - "change-me" + - "changethis" + - "xxxxxx" + - "example" + - "placeholder" + - "dummy" + - "redacted" + - "your_api_key" + - "your-api-key" + - "yourapikey" + - "insert_secret_here" + - "todo" + - "fixme" + - "null" + - "none" + - "false" diff --git a/go.mod b/go.mod index 7c069d84336f..4f12a6394d9b 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/trufflesecurity/trufflehog/v3 go 1.25.0 -toolchain go1.25.5 +toolchain go1.25.10 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.8 @@ -16,36 +16,35 @@ require ( github.com/TheZeroSlave/zapsentry v1.23.0 github.com/adrg/strutil v0.3.1 github.com/alecthomas/kingpin/v2 v2.4.0 - github.com/avast/apkparser v0.0.0-20250626104540-d53391f4d69d - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.31.7 - github.com/aws/aws-sdk-go-v2/credentials v1.18.11 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 - github.com/aws/aws-sdk-go-v2/service/sns v1.38.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.38.3 - github.com/aws/smithy-go v1.24.2 - github.com/aymanbagabas/go-osc52 v1.2.1 + github.com/avast/apkparser v0.0.0-20260423123151-7fcaee440f68 + github.com/aws/aws-sdk-go-v2 v1.43.7 + github.com/aws/aws-sdk-go-v2/config v1.32.38 + github.com/aws/aws-sdk-go-v2/credentials v1.19.37 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.44 + github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 + github.com/aws/aws-sdk-go-v2/service/sns v1.42.7 + github.com/aws/aws-sdk-go-v2/service/sts v1.45.7 + github.com/aws/smithy-go v1.27.9 + github.com/aymanbagabas/go-osc52 v1.2.2 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 github.com/brianvoe/gofakeit/v7 v7.6.0 github.com/charmbracelet/bubbles v0.18.0 - github.com/charmbracelet/bubbletea v1.3.6 + github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/glamour v0.10.0 github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/couchbase/gocb/v2 v2.11.0 github.com/crewjam/rfc5424 v0.1.0 github.com/csnewman/dextk v0.3.0 - github.com/docker/docker v28.5.2+incompatible github.com/dustin/go-humanize v1.0.1 github.com/elastic/go-elasticsearch/v8 v8.17.1 github.com/envoyproxy/protoc-gen-validate v1.3.0 github.com/fatih/color v1.18.0 github.com/felixge/fgprof v0.9.5 - github.com/gabriel-vasile/mimetype v1.4.10 + github.com/gabriel-vasile/mimetype v1.4.15 github.com/getsentry/sentry-go v0.32.0 github.com/go-errors/errors v1.5.1 - github.com/go-git/go-git/v5 v5.17.1 + github.com/go-git/go-git/v5 v5.19.2 github.com/go-logr/logr v1.4.3 github.com/go-logr/zapr v1.3.0 github.com/go-redis/redis v6.15.9+incompatible @@ -59,11 +58,13 @@ require ( github.com/googleapis/gax-go/v2 v2.16.0 github.com/hashicorp/go-retryablehttp v0.7.8 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/jackc/pgx/v5 v5.9.2 github.com/jedib0t/go-pretty/v6 v6.6.8 github.com/jlaffaye/ftp v0.2.0 github.com/joho/godotenv v1.5.1 github.com/jpillora/overseer v1.1.6 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 + github.com/klauspost/compress v1.18.6 github.com/klauspost/pgzip v1.2.6 github.com/kylelemons/godebug v1.1.0 github.com/lestrrat-go/jwx/v3 v3.0.12 @@ -74,40 +75,41 @@ require ( github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d github.com/microsoft/go-mssqldb v1.8.2 github.com/mitchellh/go-ps v1.0.0 + github.com/moby/moby/client v0.4.1 github.com/muesli/reflow v0.3.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/paulbellamy/ratecounter v0.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.5 - github.com/rabbitmq/amqp091-go v1.10.0 + github.com/rabbitmq/amqp091-go v1.13.0 github.com/repeale/fp-go v0.11.1 github.com/sassoftware/go-rpmutils v0.4.0 github.com/schollz/progressbar/v3 v3.17.1 github.com/sendgrid/sendgrid-go v3.16.1+incompatible github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 github.com/shuheiktgw/go-travis v0.3.1 - github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 - github.com/stretchr/testify v1.11.1 - github.com/testcontainers/testcontainers-go v0.34.0 - github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0 - github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0 - github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0 - github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0 - github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0 + github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed + github.com/stretchr/testify v1.12.1 + github.com/testcontainers/testcontainers-go v0.42.0 + github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.42.0 + github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0 + github.com/testcontainers/testcontainers-go/modules/mssql v0.42.0 + github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.1-0.20260423004847-9fc0246c3859 github.com/trufflesecurity/disk-buffer-reader v0.2.1 - github.com/trufflesecurity/ldap-verify v0.0.0-20260401205135-1cad02b26426 - github.com/wasilibs/go-re2 v1.9.0 + github.com/trufflesecurity/ldap-verify v0.0.0-20260924192009-3142cc951070 + github.com/wasilibs/go-re2 v1.12.0 github.com/xo/dburl v0.23.8 gitlab.com/gitlab-org/api/client-go v1.12.0 - go.mongodb.org/mongo-driver v1.17.4 + go.mongodb.org/mongo-driver v1.17.7 go.uber.org/automaxprocs v1.6.0 go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.46.0 - golang.org/x/net v0.48.0 + golang.org/x/crypto v0.53.0 + golang.org/x/net v0.56.0 golang.org/x/oauth2 v0.34.0 - golang.org/x/sync v0.19.0 - golang.org/x/text v0.32.0 + golang.org/x/sync v0.21.0 + golang.org/x/text v0.39.0 golang.org/x/time v0.14.0 google.golang.org/api v0.259.0 google.golang.org/protobuf v1.36.11 @@ -115,7 +117,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 pault.ag/go/debian v0.18.0 - pgregory.net/rapid v1.1.0 + pgregory.net/rapid v1.2.0 sigs.k8s.io/yaml v1.4.0 ) @@ -127,10 +129,10 @@ require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.3 // indirect cloud.google.com/go/monitoring v1.24.3 // indirect - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Azure/go-ntlmssp v0.1.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Azure/go-ntlmssp v0.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect @@ -143,28 +145,28 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.29.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.7 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.0 // indirect github.com/bodgit/windows v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect - github.com/charmbracelet/x/ansi v0.9.3 // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13 // indirect github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf // indirect github.com/charmbracelet/x/term v0.2.1 // indirect @@ -180,29 +182,30 @@ require ( github.com/couchbase/goprotostellar v1.0.2 // indirect github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect - github.com/cyphar/filepath-securejoin v0.4.1 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/docker/cli v29.3.1+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.8 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect @@ -223,19 +226,21 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jpillora/s3 v1.1.4 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc/v3 v3.0.1 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magiconair/properties v1.8.7 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -243,14 +248,14 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.1.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/api v1.54.2 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect - github.com/moby/term v0.5.0 // indirect + github.com/moby/term v0.5.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect @@ -260,10 +265,9 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect - github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -271,22 +275,21 @@ require ( github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect - github.com/shirou/gopsutil/v3 v3.23.12 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shirou/gopsutil/v4 v4.26.4 // indirect github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - github.com/tetratelabs/wazero v1.9.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/tetratelabs/wazero v1.12.0 // indirect github.com/therootcompany/xz v1.0.1 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/trufflesecurity/touchfile v0.1.1 // indirect github.com/ulikunitz/xz v0.5.15 // indirect github.com/vbatts/tar-split v0.12.1 // indirect - github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -297,11 +300,11 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yuin/goldmark v1.7.8 // indirect github.com/yuin/goldmark-emoji v1.0.5 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect @@ -309,10 +312,11 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/term v0.38.0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect diff --git a/go.sum b/go.sum index 28d7746b281f..0ac2d5056afc 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ cloud.google.com/go/storage v1.56.1 h1:n6gy+yLnHn0hTwBFzNn8zJ1kqWfR91wzdM8hjRF4w cloud.google.com/go/storage v1.56.1/go.mod h1:C9xuCZgFl3buo2HZU/1FncgvvOgTAs/rnh4gF4lMg0s= cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= @@ -55,10 +55,10 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= -github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw= +github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= @@ -104,50 +104,50 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/avast/apkparser v0.0.0-20250626104540-d53391f4d69d h1:PGSn2pnK/u5ZBompy83R6Wo4BqLYp3dX43QWDoPv7TA= -github.com/avast/apkparser v0.0.0-20250626104540-d53391f4d69d/go.mod h1:3F9A8btIerUcuy7Fmno+g/nIk4ELKJ6NCs2/KK1bvLs= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.31.7 h1:zS1O6hr6t0nZdBCMFc/c9OyZFyLhXhf/B2IZ9Y0lRQE= -github.com/aws/aws-sdk-go-v2/config v1.31.7/go.mod h1:GpHmi1PQDdL5pP4JaB00pU0ek4EXVcYH7IkjkUadQmM= -github.com/aws/aws-sdk-go-v2/credentials v1.18.11 h1:1Fnb+7Dk96/VYx/uYfzk5sU2V0b0y2RWZROiMZCN/Io= -github.com/aws/aws-sdk-go-v2/credentials v1.18.11/go.mod h1:iuvn9v10dkxU4sDgtTXGWY0MrtkEcmkUmjv4clxhuTc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 h1:Is2tPmieqGS2edBnmOJIbdvOA6Op+rRpaYR60iBAwXM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7/go.mod h1:F1i5V5421EGci570yABvpIXgRIBPb5JM+lSkHF6Dq5w= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5 h1:fSuJX/VBJKufwJG/szWgUdRJVyRiEQDDXNh/6NPrTBg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5/go.mod h1:LvN0noQuST+3Su55Wl++BkITpptnfN9g6Ohkv4zs9To= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/sns v1.38.2 h1:Djc2m7mTPuizL1iMxJfMc209PDy2AqiN1AXrtq/rBdY= -github.com/aws/aws-sdk-go-v2/service/sns v1.38.2/go.mod h1:kHMCS+JDWKuKSDP9J/v3dlV2S9zNBKbXzaLy/kHSdEE= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.2 h1:rcoTaYOhGE/zfxE1uR6X5fvj+uKkqeCNRE0rBbiQM34= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.2/go.mod h1:Ql6jE9kyyWI5JHn+61UT/Y5Z0oyVJGmgmJbZD5g4unY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3 h1:BSIfeFtU9tlSt8vEYS7KzurMoAuYzYPWhcZiMtxVf2M= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3/go.mod h1:XclEty74bsGBCr1s0VSaA11hQ4ZidK4viWK7rRfO88I= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.3 h1:yEiZ0ztgji2GsCb/6uQSITXcGdtmWMfLRys0jJFiUkc= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.3/go.mod h1:Z+Gd23v97pX9zK97+tX4ppAgqCt3Z2dIXB02CtBncK8= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= -github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E= -github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/avast/apkparser v0.0.0-20260423123151-7fcaee440f68 h1:+7UZ0vN+mEIzkjcgC5ZUzTV+ikKqCrfK06OyVCQyzAo= +github.com/avast/apkparser v0.0.0-20260423123151-7fcaee440f68/go.mod h1:3F9A8btIerUcuy7Fmno+g/nIk4ELKJ6NCs2/KK1bvLs= +github.com/aws/aws-sdk-go-v2 v1.43.7 h1:msCzvkeYJA9ehbV8mRRmkZLo/zJg/+yDVLNtflg83hQ= +github.com/aws/aws-sdk-go-v2 v1.43.7/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM= +github.com/aws/aws-sdk-go-v2/config v1.32.38 h1:n4yPHBjtQ3BrIIUyk0/LAqf/BL2iv0Tw6XZcMRzM0ps= +github.com/aws/aws-sdk-go-v2/config v1.32.38/go.mod h1:dencYsOS1R7rBy8zehCvwBYzdxxL4Q/nRK7In03wjN8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.37 h1:FJ8Iz4/xISMB/rwLlgfWujfGDFWr0oneQgtA6KPcYLY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.37/go.mod h1:Q6pWOgVUp49x4g5QVi29wHofUoICnZ+Zq4jHbRN/7ec= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.38 h1:Nqo2jU1wz5rnBM9XQyXfVD1RP8txkbP3EDx8hR/hbCE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.38/go.mod h1:PzJFHhjR2vWFKHe8HmY5Lxhvwyxnr5MERtk0nDxWNbk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.44 h1:SdZTMore4w293Z85VCaTu/RnUfQvpixYThMOLcjqKmI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.44/go.mod h1:Vm/BzUBXbYPing3k/eayNkjitWH0Ml4Ds8mYgMWnEUc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38 h1:MBMg0zJ6i4TkAJ0dVFLKKn2cOkY6FkicmUDM67BRr6g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38/go.mod h1:9MWuJbyiUyj6eA7W1/zm1zuePDPSB3g+xcgRQeMWsXc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38 h1:lHm4jPf3k1Lz5ZWc+Vcn3MKVwym+26kWCba9FkJ4f0Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38/go.mod h1:Rn+P2XR+FbyZzjmWKjg/KUZNxmGfr5oZwh5jQiE+CzI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39 h1:vo4xvMRs/F6h1E52qsgLqCQgWIQXgIJUauG6rlZEh4U= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39/go.mod h1:jB03R1ij/A+OE2e1dz6vgj076gd7vlYcfstAzj3HcnU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 h1:uZOinZb+h7lZw8IYzP1z1IuEnueB76/EFkcf/fEW4Ag= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31/go.mod h1:NRtwAM/p5VRt03TlEUs0pH3TeWamWdf4YyJpSrzPYLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 h1:H/5TI1jqaHsNoDQ60UwvPvJBg4GURkinXI3Qga29t2w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38/go.mod h1:PTVFf+XH++7NJOky+RLBYQx0QA5NcaeEYFQ2fsi0nwo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 h1:HLPAVrlLDaN2boN0xJx7MgaQDNEO3Q+c9L6kl/8m47Q= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39/go.mod h1:Pg/dVfsNkm1hsIDK/gMvCKtmyNfNTV12mrgHqVE/6Oo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 h1:IKoCZqfWfZzSBi16QFQ+QcbQ3LRQ7QgB1S5tDAyPBQQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3/go.mod h1:RBpRcXiM4s2pOInVs32GsBonnje+fiAj4mcrStRmlCA= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.7 h1:YcczQ6zNH/ojIzD/ikDrO+RfW06wmdMp18d4NH5hXY4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.7/go.mod h1:nl9RVnb9ulgAYzOkjLq1NyFxmWcnH2maCUEuOdESy98= +github.com/aws/aws-sdk-go-v2/service/sns v1.42.7 h1:uN4m1MGl4XAbm2D0+ZuAS29PQt9qatLPAj6rabY9yUs= +github.com/aws/aws-sdk-go-v2/service/sns v1.42.7/go.mod h1:b1u4gpEJLmL+icKTdEM9+9oC4alL+sKrVi4T/IbGrX8= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.7 h1:P+bMNiA93gyuYT3Oh+4dWtvrnGcu2bd9Uy5hRJM8BNo= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.7/go.mod h1:zy+397isDFLvleg9H18Zq2MGzMso7uKyJyzR7DWSgFk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.7 h1:WWkehGZ4nWtOKLMy0yi8+RqzzVqAGe60hGaxwF06JAw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.7/go.mod h1:T8AI4SbQYm9ybcVmki2T3n7Qg1g3kfWoeQlNwNYOyO8= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.7 h1:yU/9y2r7s9kSUPbHXbpQTa4LA8kt+CMgpu1OBrhx8p4= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.7/go.mod h1:0lQTDEBArMevQXpxu443LVGjKxxEeSsSnrw9n8YiTMg= +github.com/aws/smithy-go v1.27.9 h1:flT/ACSU1ksz3V+8wj8kN8DOB9tsc/ggWPTJXIieRpw= +github.com/aws/smithy-go v1.27.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aymanbagabas/go-osc52 v1.2.2 h1:NT7wkhEhPTcKnBCdPi9djmyy9L3JOL4+3SsfJyqptCo= +github.com/aymanbagabas/go-osc52 v1.2.2/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= @@ -169,23 +169,23 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 h1:B91r9bHtXp/+XRgS5aZm6ZzTdz github.com/bradleyfalzon/ghinstallation/v2 v2.16.0/go.mod h1:OeVe5ggFzoBnmgitZe/A+BqGOnv1DvU/0uiLQi1wutM= github.com/brianvoe/gofakeit/v7 v7.6.0 h1:M3RUb5CuS2IZmF/cP+O+NdLxJEuDAZxNQBwPbbqR6h4= github.com/brianvoe/gofakeit/v7 v7.6.0/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= -github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGsK6et5taU= -github.com/charmbracelet/bubbletea v1.3.6/go.mod h1:oQD9VCRQFF8KplacJLo28/jofOI2ToOfGYeFgBBxHOc= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= -github.com/charmbracelet/x/ansi v0.9.3 h1:BXt5DHS/MKF+LjuK4huWrC6NCvHtexww7dMayh6GXd0= -github.com/charmbracelet/x/ansi v0.9.3/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= +github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ= +github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= @@ -235,14 +235,14 @@ github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 h1:lh github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28/go.mod h1:o7T431UOfFVHDNvMBUmUxpHnhivwv7BziUao/nMl81E= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/crewjam/rfc5424 v0.1.0 h1:MSeXJm22oKovLzWj44AHwaItjIMUMugYGkEzfa831H8= github.com/crewjam/rfc5424 v0.1.0/go.mod h1:RCi9M3xHVOeerf6ULZzqv2xOGRO/zYaVUeRyPnBW3gQ= github.com/csnewman/dextk v0.3.0 h1:gigNZlZRNfCuARV7depunRlafEAzGhyvgBQo1FT3/0M= github.com/csnewman/dextk v0.3.0/go.mod h1:FcDoI3258ea0KPQogyv4iazQRGcLFNOW+I4pHBUfNO0= -github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= -github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -261,8 +261,8 @@ github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaft github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= @@ -270,6 +270,8 @@ github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elastic/elastic-transport-go/v8 v8.6.1 h1:h2jQRqH6eLGiBSN4eZbQnJLtL4bC5b4lfVFRjw2R4e4= github.com/elastic/elastic-transport-go/v8 v8.6.1/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/go-elasticsearch/v8 v8.17.1 h1:bOXChDoCMB4TIwwGqKd031U8OXssmWLT3UrAr9EGs3Q= @@ -302,24 +304,24 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= -github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= +github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= github.com/getsentry/sentry-go v0.32.0 h1:YKs+//QmwE3DcYtfKRH8/KyOOF/I6Qnx7qYGNHCGmCY= github.com/getsentry/sentry-go v0.32.0/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= -github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= -github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ= +github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= -github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.17.1 h1:WnljyxIzSj9BRRUlnmAU35ohDsjRK0EKmL0evDqi5Jk= -github.com/go-git/go-git/v5 v5.17.1/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -333,8 +335,9 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= @@ -390,7 +393,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -456,12 +458,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= -github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.6.8 h1:JnnzQeRz2bACBobIaa/r+nqjvws4yEhcmaZ4n1QzsEc= @@ -484,9 +486,11 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKMYMj0FJRCF6tQubUuQ02GM= github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -522,10 +526,10 @@ github.com/lrstanley/bubblezone v0.0.0-20250404061050-e13639e27357 h1:DxFncLGTrD github.com/lrstanley/bubblezone v0.0.0-20250404061050-e13639e27357/go.mod h1:S5etECMx+sZnW0Gm100Ma9J1PgVCTgNyFaqGu2b08b4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marusama/semaphore/v2 v2.5.0 h1:o/1QJD9DBYOWRnDhPwDVAXQn6mQYD0gZaS1Tpx6DJGM= github.com/marusama/semaphore/v2 v2.5.0/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ= @@ -539,6 +543,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d h1:Vw3f39TqFSQLA+OyW+8SouppHTYzX8/fDv6Ao8uj3Ho= github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= @@ -553,10 +559,14 @@ github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= -github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= +github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= +github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= @@ -565,8 +575,8 @@ github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -611,8 +621,8 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= -github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -621,10 +631,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -636,8 +644,8 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= -github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA= +github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/repeale/fp-go v0.11.1 h1:Q/e+gNyyHaxKAyfdbBqvip3DxhVWH453R+kthvSr9Mk= github.com/repeale/fp-go v0.11.1/go.mod h1:4KrwQJB1VRY+06CA+jTc4baZetr6o2PeuqnKr5ybQUc= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -662,22 +670,18 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= -github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shirou/gopsutil/v4 v4.26.4 h1:B4SXVbcwTyrocPHEmWBC4uCYr4Xcu3MK1TXqbprAOWY= +github.com/shirou/gopsutil/v4 v4.26.4/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY= github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg= -github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M= -github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= +github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed h1:KT7hI8vYXgU0s2qaMkrfq9tCA1w/iEPgfredVP+4Tzw= +github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -692,8 +696,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -704,32 +708,32 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= -github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= -github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0 h1:BBwJUs9xBpt1uOfO+yAr2pYW75MsyzuO/o70HTPnhe4= -github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0/go.mod h1:OqhRGYR+5VG0Dw506F6Ho9I4YG1kB+o9uPTKC0uPUA8= -github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0 h1:o3bgcECyBFfMwqexCH/6vIJ8XzbCffCP/Euesu33rgY= -github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0/go.mod h1:ljLR42dN7k40CX0dp30R8BRIB3OOdvr7rBANEpfmMs4= -github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0 h1:4Pf7ILuLnxhpeTgQfKzEMPuMQhasU3VaYer9l5HrQ3s= -github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0/go.mod h1:L2eMWZ49X0XjewabzJ6TXSY5z4SAWM/2WOBqlIxYFD8= -github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0 h1:Tqz17mGXjPORHFS/oBUGdeJyIsZXLsVVHRhaBqhewGI= -github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0/go.mod h1:hDpm3DLfjo7rd6232wWflEBDGr6Ow9ys43mJTiJwWx8= -github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0 h1:c51aBXT3v2HEBVarmaBnsKzvgZjC5amn0qsj8Naqi50= -github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0/go.mod h1:EWP75ogLQU4M4L8U+20mFipjV4WIR9WtlMXSB6/wiuc= -github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= -github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.42.0 h1:GzdsTn4x0wTkdmRl2EFk0IniDGRoPCKPW/NWi9EmKs8= +github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.42.0/go.mod h1:5GEeZpWraS5ewjGXpMYbFodJjT+vWxuKRZ3fNXQOGRA= +github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0 h1:jX10Aprgf1L+Ov+KxcheZ/1JXdiJ/3wdevfWFSkxm6s= +github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0/go.mod h1:Ph+xH0hAC6djPFTjPgLa3VmSfE4h82kzVIKxTj3n2o4= +github.com/testcontainers/testcontainers-go/modules/mssql v0.42.0 h1:xRgMG7xUAEFTDl37n7qyYopDwv2dzPMHdKM4+fzGTRo= +github.com/testcontainers/testcontainers-go/modules/mssql v0.42.0/go.mod h1:JGteeyk5pkInLpBn8Q3RtD/dpdcxBk8QNzr9/3G3PRo= +github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0 h1:Yhv1k7vDpyzZePntg5R5Oj4ZMCyWpAfpJeRu1ROsgiU= +github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0/go.mod h1:Z7SCTuiZlghAdRjkv3Ir0iXJKC2T2avbtxLR0DRe+ng= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.1-0.20260423004847-9fc0246c3859 h1:FQ0GtddzRvW5JpBtYSlhHuZDuIJ8ftjfeZfVDcqdCmU= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.1-0.20260423004847-9fc0246c3859/go.mod h1:wbXmKjW6MkKFaRP6YWwVR9ZDeq1tReh0wdWeutuDv+o= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/trufflesecurity/disk-buffer-reader v0.2.1 h1:K9nNpX3xeWT2E6YRjlcc1X5c1NjgV9JS5T9aw2FjA8Q= github.com/trufflesecurity/disk-buffer-reader v0.2.1/go.mod h1:uYwTCdxzV0o+qaeBMxflOsq4eu2WjrE46qGR2e80O9Y= -github.com/trufflesecurity/ldap-verify v0.0.0-20260401205135-1cad02b26426 h1:vCSA2SQHtO1k6F6IIwHisjgkegy78w6w10QsPbdXppo= -github.com/trufflesecurity/ldap-verify v0.0.0-20260401205135-1cad02b26426/go.mod h1:ziPWHgmsF3s1VmgP0eTrR0i7fHxlSOqch60dfjHUxHw= +github.com/trufflesecurity/ldap-verify v0.0.0-20260924192009-3142cc951070 h1:Poq+Z6/uKeNJwSKE8ErQlIVf2b3L3Zyzpw6mQbJ784A= +github.com/trufflesecurity/ldap-verify v0.0.0-20260924192009-3142cc951070/go.mod h1:VQbB7BbEEjTfPqN9dT/3jksvk1p1uj5YQZLMZ8Unu6E= github.com/trufflesecurity/overseer v1.2.8 h1:VXlWPiwYaQmwNxY2W1rVulEAG9O6iF1S0LX3wionWYM= github.com/trufflesecurity/overseer v1.2.8/go.mod h1:Dt6Y9LFpM+C/3rRWpy4//4iS5qrbb0pL3XvZqMd4zhg= github.com/trufflesecurity/touchfile v0.1.1 h1:Snhd5VEa8Cxd+D60nvLEj2kVeb1omY2tWwnhDhjTqdo= @@ -741,10 +745,10 @@ github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXV github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= -github.com/wasilibs/go-re2 v1.9.0 h1:kjAd8qbNvV4Ve2Uf+zrpTCrDHtqH4dlsRXktywo73JQ= -github.com/wasilibs/go-re2 v1.9.0/go.mod h1:0sRtscWgpUdNA137bmr1IUgrRX0Su4dcn9AEe61y+yI= -github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= -github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= +github.com/wasilibs/go-re2 v1.12.0 h1:sq3A6ZOqT90HYY25MD5/cG8Xv6uT2AhmPgBfkCfhp10= +github.com/wasilibs/go-re2 v1.12.0/go.mod h1:2W+7GrrdO4NHv7ITHz8Yy3a1IxzjZCk8JR5zgTprxZs= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -773,12 +777,14 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC4cIk= github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= gitlab.com/gitlab-org/api/client-go v1.12.0 h1:vYeraq+4eC/5Scir5nWve3LPxLoY5rgW2qRNUgjKS2k= gitlab.com/gitlab-org/api/client-go v1.12.0/go.mod h1:adtVJ4zSTEJ2fP5Pb1zF4Ox1OKFg0MH43yxpb0T0248= -go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= -go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver v1.17.7 h1:a9w+U3Vt67eYzcfq3k/OAv284/uUUkL0uP75VE5rCOU= +go.mongodb.org/mongo-driver v1.17.7/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver/v2 v2.3.0 h1:sh55yOXA2vUjW1QYw/2tRlHSQViwDyPnW61AwpZ4rtU= +go.mongodb.org/mongo-driver/v2 v2.3.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -789,8 +795,8 @@ go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK2 go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= @@ -823,6 +829,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -832,8 +840,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -842,8 +850,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= -golang.org/x/exp v0.0.0-20250813145105-42675adae3e6/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -863,8 +871,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -886,8 +894,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -903,8 +911,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -941,18 +949,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -962,8 +968,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= @@ -1092,8 +1098,8 @@ pault.ag/go/debian v0.18.0 h1:nr0iiyOU5QlG1VPnhZLNhnCcHx58kukvBJp+dvaM6CQ= pault.ag/go/debian v0.18.0/go.mod h1:JFl0XWRCv9hWBrB5MDDZjA5GSEs1X3zcFK/9kCNIUmE= pault.ag/go/topsort v0.1.1 h1:L0QnhUly6LmTv0e3DEzbN2q6/FGgAcQvaEw65S53Bg4= pault.ag/go/topsort v0.1.1/go.mod h1:r1kc/L0/FZ3HhjezBIPaNVhkqv8L0UJ9bxRuHRVZ0q4= -pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= -pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/hack/checksecretparts/README.md b/hack/checksecretparts/README.md index 4f8a92832855..7acfe0ca279e 100644 --- a/hack/checksecretparts/README.md +++ b/hack/checksecretparts/README.md @@ -9,22 +9,9 @@ For each directory under `pkg/detectors/` (recursing into subpackages): 1. Find every composite literal of the form `detectors.Result{...}` or `&detectors.Result{...}` in non-test `.go` files. -2. If the package does not mention `SecretParts` anywhere (neither in the - literal nor in a later `x.SecretParts = ...` assignment), emit a warning +2. If the package does not mention `SecretParts` anywhere, emit a warning for each construction site. -Test files (`_test.go`) are ignored on both sides — construction sites in -tests are not flagged, and `SecretParts` references in tests do not suppress -findings, because some tests zero the field for comparison. - -## Why warning-only - -This check ships as **warning-only** because ~907 existing detectors do not -yet populate `SecretParts` (see the SecretParts design doc, step C). -Hard-failing today would block every unrelated PR. The check is wired into -CI with `continue-on-error: true` so the findings are visible without -gating merges. - ## Running locally ```sh @@ -41,8 +28,7 @@ go run ./hack/checksecretparts -fail ## Flipping warning → fail -Once step C is complete and every detector populates `SecretParts`, make -this check gating: +Once every detector populates `SecretParts`, make this check gating: 1. In `.github/workflows/lint.yml`, drop `continue-on-error: true` from the `checksecretparts` job and change the run step to pass `-fail`. @@ -53,7 +39,3 @@ this check gating: - It is a syntactic check. It matches `detectors.Result` by selector-expr name; packages that rename the import (`d "...detectors"`) would not be caught. No such rename exists in the current codebase. -- It does not verify that `SecretParts` is populated on every code path — - only that the package touches the field at all. A finer-grained - dataflow check is deliberately out of scope; the rough check is enough - to surface unmigrated detectors. diff --git a/hack/checksecretparts/check.go b/hack/checksecretparts/check.go index c1995ba6ace5..c0983d3c81f5 100644 --- a/hack/checksecretparts/check.go +++ b/hack/checksecretparts/check.go @@ -21,12 +21,7 @@ type Finding struct { } // CheckPackageDir runs the SecretParts check on a single directory. It returns -// one Finding per detectors.Result{} construction site in the directory, -// filtered so that packages which mention SecretParts anywhere produce no -// findings. Test files (_test.go) are ignored on both sides: construction -// sites in them are not reported, and references in them do not suppress -// findings (test files commonly zero the field for comparison — see -// pkg/detectors/gitlab/v1/gitlab_integration_test.go). +// one Finding per detectors.Result{} construction site in the directory. func CheckPackageDir(dir string) ([]Finding, error) { fset := token.NewFileSet() entries, err := os.ReadDir(dir) @@ -61,22 +56,12 @@ func CheckPackageDir(dir string) ([]Finding, error) { // and returns findings. It is separated from CheckPackageDir so that tests can // drive it with synthetic ASTs. func checkFiles(fset *token.FileSet, dir string, files []*ast.File) []Finding { - var ( - constructions []token.Position - hasSecretParts bool - ) + var constructions []token.Position for _, f := range files { - if fileReferencesSecretParts(f) { - hasSecretParts = true - } constructions = append(constructions, findResultConstructions(fset, f)...) } - if hasSecretParts || len(constructions) == 0 { - return nil - } - sort.Slice(constructions, func(i, j int) bool { if constructions[i].Filename != constructions[j].Filename { return constructions[i].Filename < constructions[j].Filename @@ -155,30 +140,3 @@ func hasSecretPartsKey(lit *ast.CompositeLit) bool { } return false } - -// fileReferencesSecretParts returns true if the file mentions the identifier -// "SecretParts" in any form: a composite-literal key, a selector expression -// (x.SecretParts), or a bare identifier. The rationale is that if a detector -// package touches SecretParts at all — whether on the construction site or in -// a later assignment — it has been migrated; the check's job is to find -// packages that never touch it. -func fileReferencesSecretParts(f *ast.File) bool { - found := false - ast.Inspect(f, func(n ast.Node) bool { - if found { - return false - } - switch x := n.(type) { - case *ast.Ident: - if x.Name == "SecretParts" { - found = true - } - case *ast.SelectorExpr: - if x.Sel != nil && x.Sel.Name == "SecretParts" { - found = true - } - } - return !found - }) - return found -} diff --git a/hack/checksecretparts/check_test.go b/hack/checksecretparts/check_test.go index 7d390c4a638f..857cad49e0d1 100644 --- a/hack/checksecretparts/check_test.go +++ b/hack/checksecretparts/check_test.go @@ -49,7 +49,7 @@ func FromData() detectors.Result { wantLen: 0, }, { - name: "SecretParts assigned later is accepted", + name: "SecretParts assigned later is not accepted", files: map[string]string{ "det.go": `package det @@ -65,7 +65,7 @@ func FromData() detectors.Result { } `, }, - wantLen: 0, + wantLen: 1, }, { name: "no detectors.Result construction is a no-op", diff --git a/hack/checksecretparts/main.go b/hack/checksecretparts/main.go index 209decce9973..e300b32f156b 100644 --- a/hack/checksecretparts/main.go +++ b/hack/checksecretparts/main.go @@ -33,9 +33,9 @@ func main() { flag.BoolVar(&failOnFindings, "fail", false, "exit 1 if any findings are reported (default: warning-only)") flag.BoolVar(&quiet, "quiet", false, "suppress the summary line when no findings are reported") flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [dir ...]\n", os.Args[0]) - fmt.Fprintln(flag.CommandLine.Output(), "\nFinds detector packages that construct detectors.Result without setting SecretParts.") - fmt.Fprintln(flag.CommandLine.Output(), "\nFlags:") + _, _ = fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [dir ...]\n", os.Args[0]) + _, _ = fmt.Fprintln(flag.CommandLine.Output(), "\nFinds detector packages that construct detectors.Result without setting SecretParts.") + _, _ = fmt.Fprintln(flag.CommandLine.Output(), "\nFlags:") flag.PrintDefaults() } flag.Parse() @@ -62,7 +62,7 @@ func main() { } for _, f := range findings { - fmt.Printf("%s: warning: detectors.Result constructed without SecretParts (no reference to SecretParts anywhere in package)\n", f.Position) + fmt.Printf("%s: warning: detectors.Result constructed without SecretParts\n", f.Position) } if len(findings) > 0 { diff --git a/hack/docs/Adding_Detectors_Internal.md b/hack/docs/Adding_Detectors_Internal.md index c1951feb494d..a20a1f74452f 100644 --- a/hack/docs/Adding_Detectors_Internal.md +++ b/hack/docs/Adding_Detectors_Internal.md @@ -15,7 +15,7 @@ The purpose of Secret Detectors is to discover secrets with exceptionally high s - [Sourcing Guidelines](#sourcing-guidelines) - [Development Guidelines](#development-guidelines) - [Development Dependencies](#development-dependencies) - - [Creating a new Secret Scanner](#creating-a-new-secret-detector) + - [Creating a new Secret Scanner](#creating-a-new-secret-scanner) - [Populating SecretParts](#populating-secretparts) - [Addendum](#addendum) - [Verification indeterminacy](#verification-indeterminacy) @@ -35,7 +35,7 @@ If you think that something should be included outside of these guidelines, plea ### Development Guidelines - When reasonable, favor using the `net/http` library to make requests instead of bringing in another library. -- Use the [`common.SaneHttpClient`](pkg/common/http.go) for the `http.Client` whenever possible. +- Use the [`common.SaneHttpClient`](/pkg/common/http.go) for the `http.Client` whenever possible. - We recommend an editor with gopls integration (such as Vscode with Go plugin) for benefits like easily running tests, autocompletion, linting, type checking, etc. ### Development Dependencies @@ -179,7 +179,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager ``` Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb). -4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go). +4. Access the secret value as shown in the [example code](/pkg/detectors/heroku/v1/heroku_integration_test.go). ### Setting up Google Cloud SDK diff --git a/hack/docs/Adding_Detectors_external.md b/hack/docs/Adding_Detectors_external.md index cf338db9fc2a..91575c32dbdb 100644 --- a/hack/docs/Adding_Detectors_external.md +++ b/hack/docs/Adding_Detectors_external.md @@ -57,7 +57,7 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or 4. Update the existing detector in DefaultDetectors in `/pkg/engine/defaults/defaults.go` -5. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner) +5. Proceed from step 3 of [Creating a new Secret Detector](#creating-a-new-secret-detector) ### Creating a new Secret Detector diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go index fe401aa9e991..b2c46eec898c 100644 --- a/hack/snifftest/main.go +++ b/hack/snifftest/main.go @@ -219,7 +219,7 @@ func main() { logFatal(err, "error scanning repo") } logger.Info("scanned repo", "repo", r) - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() }(repo) } diff --git a/main.go b/main.go index 6ae40d127a0b..944b67921c7d 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ var ( jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool() jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool() gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool() + sarifOut = cli.Flag("sarif", "Output in SARIF format for upload to GitHub code scanning (e.g. via github/codeql-action/upload-sarif).").Bool() concurrency = cli.Flag("concurrency", "Number of concurrent workers.").PlaceHolder("N").Int() noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool() onlyVerified = cli.Flag("only-verified", "Only output verified results.").Hidden().Bool() @@ -66,6 +67,7 @@ var ( allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64() + noIgnoreTag = cli.Flag("no-ignore-tag", "Report results even if the line has a 'trufflehog:ignore' comment.").Bool() scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool() maxDecodeDepth = cli.Flag("max-decode-depth", "Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).").Default("5").Int() compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool() @@ -88,11 +90,13 @@ var ( noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool() // Add feature flags - forceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool() - forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool() - gitCloneTimeout = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration() - skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool() - userAgentSuffix = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String() + forceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool() + forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool() + gitCloneTimeout = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration() + gitLowMemoryScan = cli.Flag("git-low-memory-scan", "Reduce memory use for git scanning.").Hidden().Bool() + skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool() + userAgentSuffix = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String() + dropUnverifiedJWTResults = cli.Flag("drop-unverified-jwt-results", "Drop unverified results without any verification errors from the JWT detector.").Bool() gitScan = cli.Command("git", "Find credentials in git repositories.") gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String() @@ -104,7 +108,7 @@ var ( gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int() gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool() gitClonePath = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String() - gitNoCleanup = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool() + gitNoCleanup = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool() gitTrustLocalGitConfig = gitScan.Flag("trust-local-git-config", "Trust local git config.").Bool() _ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool() _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool() @@ -128,8 +132,9 @@ var ( githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32() githubAuthInUrl = githubScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool() githubClonePath = githubScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String() - githubNoCleanup = githubScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool() + githubNoCleanup = githubScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool() githubIgnoreGists = githubScan.Flag("ignore-gists", "Ignore all gists in scan.").Bool() + githubExcludeArchived = githubScan.Flag("exclude-archived", "Exclude archived repositories from scan.").Bool() // GitHub Cross Fork Object Reference Experimental Feature githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.") @@ -152,8 +157,8 @@ var ( gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() gitlabScanExcludeRepos = gitlabScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() gitlabAuthInUrl = gitlabScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool() - gitlabClonePath = gitlabScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir)").String() - gitlabNoCleanup = gitlabScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool() + gitlabClonePath = gitlabScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String() + gitlabNoCleanup = gitlabScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool() filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.") filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings() @@ -174,6 +179,13 @@ var ( s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag. Incompatible with --ignore-bucket.").Strings() s3ScanIgnoreBuckets = s3Scan.Flag("ignore-bucket", "Name of S3 bucket to ignore. You can repeat this flag. Incompatible with --bucket.").Strings() s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes() + s3ScanEndpoint = s3Scan.Flag("endpoint", "Endpoint of an S3-compatible service to scan instead of AWS S3. (eg. https://s3.internal.example.com)").String() + s3ScanRegion = s3Scan.Flag("region", "Region used to sign requests. Defaults to us-east-1.").String() + + s3ScanIncludePrefixes = s3Scan.Flag("include-prefix", "Only scan objects whose key starts with this prefix. You can repeat this flag.").Strings() + s3ScanExcludePrefixes = s3Scan.Flag("exclude-prefix", "Skip objects whose key starts with this prefix. You can repeat this flag. Takes precedence over --include-prefix.").Strings() + s3ScanIncludeExtensions = s3Scan.Flag("include-extension", "Only scan objects with this file extension, written without a leading dot (eg. tf). You can repeat this flag. Incompatible with --exclude-extension.").Strings() + s3ScanExcludeExtensions = s3Scan.Flag("exclude-extension", "Skip objects with this file extension, written without a leading dot (eg. zip). You can repeat this flag. Incompatible with --include-extension.").Strings() gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.") gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String() @@ -250,11 +262,12 @@ var ( jenkinsPassword = jenkinsScan.Flag("password", "Jenkins password").Envar("JENKINS_PASSWORD").String() jenkinsInsecureSkipVerifyTLS = jenkinsScan.Flag("insecure-skip-verify-tls", "Skip TLS verification").Envar("JENKINS_INSECURE_SKIP_VERIFY_TLS").Bool() - huggingfaceScan = cli.Command("huggingface", "Find credentials in HuggingFace datasets, models and spaces.") + huggingfaceScan = cli.Command("huggingface", "Find credentials in HuggingFace datasets, models, spaces and buckets.") huggingfaceEndpoint = huggingfaceScan.Flag("endpoint", "HuggingFace endpoint.").Default("https://huggingface.co").String() huggingfaceModels = huggingfaceScan.Flag("model", "HuggingFace model to scan. You can repeat this flag. Example: 'username/model'").Strings() huggingfaceSpaces = huggingfaceScan.Flag("space", "HuggingFace space to scan. You can repeat this flag. Example: 'username/space'").Strings() huggingfaceDatasets = huggingfaceScan.Flag("dataset", "HuggingFace dataset to scan. You can repeat this flag. Example: 'username/dataset'").Strings() + huggingfaceBuckets = huggingfaceScan.Flag("bucket", "HuggingFace bucket to scan. You can repeat this flag. Example: 'username/bucket'").Strings() huggingfaceOrgs = huggingfaceScan.Flag("org", `HuggingFace organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() huggingfaceUsers = huggingfaceScan.Flag("user", `HuggingFace user to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() huggingfaceToken = huggingfaceScan.Flag("token", "HuggingFace token. Can be provided with environment variable HUGGINGFACE_TOKEN.").Envar("HUGGINGFACE_TOKEN").String() @@ -265,9 +278,12 @@ var ( huggingfaceIgnoreModels = huggingfaceScan.Flag("ignore-models", "Models to ignore in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings() huggingfaceIgnoreSpaces = huggingfaceScan.Flag("ignore-spaces", "Spaces to ignore in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings() huggingfaceIgnoreDatasets = huggingfaceScan.Flag("ignore-datasets", "Datasets to ignore in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings() + huggingfaceIncludeBuckets = huggingfaceScan.Flag("include-buckets", "Buckets to include in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org)").Strings() + huggingfaceIgnoreBuckets = huggingfaceScan.Flag("ignore-buckets", "Buckets to ignore in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org)").Strings() huggingfaceSkipAllModels = huggingfaceScan.Flag("skip-all-models", "Skip all model scans. (Only used with --user or --org)").Bool() huggingfaceSkipAllSpaces = huggingfaceScan.Flag("skip-all-spaces", "Skip all space scans. (Only used with --user or --org)").Bool() huggingfaceSkipAllDatasets = huggingfaceScan.Flag("skip-all-datasets", "Skip all dataset scans. (Only used with --user or --org)").Bool() + huggingfaceSkipAllBuckets = huggingfaceScan.Flag("skip-all-buckets", "Skip all bucket scans. (Only used with --user or --org)").Bool() huggingfaceIncludeDiscussions = huggingfaceScan.Flag("include-discussions", "Include discussions in scan.").Bool() huggingfaceIncludePrs = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool() @@ -466,7 +482,7 @@ func run(state overseer.State, logSync func() error) { if *githubScanToken != "" { // NOTE: this kludge is here to do an authenticated shallow commit // TODO: refactor to better pass credentials - os.Setenv("GITHUB_TOKEN", *githubScanToken) + _ = os.Setenv("GITHUB_TOKEN", *githubScanToken) } if *concurrency <= 0 { @@ -505,6 +521,10 @@ func run(state overseer.State, logSync func() error) { feature.GitCloneTimeoutDuration.Store(int64(*gitCloneTimeout)) } + if *gitLowMemoryScan { + feature.UseGitLowMemoryScan.Store(true) + } + if *skipAdditionalRefs { feature.SkipAdditionalRefs.Store(true) } @@ -540,6 +560,10 @@ func run(state overseer.State, logSync func() error) { } } + if *dropUnverifiedJWTResults { + feature.DropUnverifiedJWTResults.Store(true) + } + // OSS Default APK handling on feature.EnableAPKHandler.Store(true) @@ -553,6 +577,47 @@ func run(state overseer.State, logSync func() error) { // OSS Default using github graphql api for issues, pr's and comments feature.UseGithubGraphQLAPI.Store(false) + // OSS Default Use HTML Decoder on + feature.HTMLDecoderEnabled.Store(true) + + // New detector flags + feature.PineconeDetectorEnabled.Store(true) + feature.CloudinaryDetectorEnabled.Store(true) + feature.GitLabOAuthDetectorEnabled.Store(true) + feature.FigmaV3DetectorEnabled.Store(true) + feature.SonarCloudV2DetectorEnabled.Store(true) + feature.EnigmaDetectorEnabled.Store(true) + feature.DatadogApiKeyDetectorEnabled.Store(true) + feature.TlyDetectorEnabled.Store(true) + feature.WitDetectorEnabled.Store(true) + feature.RevDetectorEnabled.Store(true) + feature.UserDetectorEnabled.Store(true) + feature.BraintrustDetectorEnabled.Store(true) + feature.PgAnalyzeReadKeyDetectorEnabled.Store(true) + feature.RedHatPyxisDetectorEnabled.Store(true) + feature.OctopusDeployDetectorEnabled.Store(true) + feature.OpenRouterDetectorEnabled.Store(true) + feature.NewRelicInsightsInsertKeyDetectorEnabled.Store(true) + feature.DuffelTokenDetectorEnabled.Store(true) + feature.ShippoDetectorEnabled.Store(true) + feature.IPInfoDetectorEnabled.Store(true) + feature.LobDetectorEnabled.Store(true) + feature.HashiCorpVaultBatchTokenDetectorEnabled.Store(true) + feature.HashiCorpVaultTokenDetectorEnabled.Store(true) + feature.CloudflareApiTokenV2DetectorEnabled.Store(true) + feature.CloudflareGlobalApiKeyV2DetectorEnabled.Store(true) + feature.DuoDetectorEnabled.Store(true) + feature.NewRelicLicenseKeyDetectorEnabled.Store(true) + feature.NewRelicBrowserKeyDetectorEnabled.Store(true) + feature.NewRelicUserKeyDetectorEnabled.Store(true) + feature.NewRelicInsightsQueryKeyDetectorEnabled.Store(true) + feature.NewRelicMobileAppTokenDetectorEnabled.Store(true) + feature.MSTeamsWebhookV2DetectorEnabled.Store(true) + feature.SolarwindsDetectorEnabled.Store(true) + feature.ResendDetectorEnabled.Store(true) + feature.WeightsAndBiasesV2DetectorEnabled.Store(true) + feature.HumioAPITokenDetectorEnabled.Store(true) + conf := &config.Config{} if *configFilename != "" { var err error @@ -586,11 +651,13 @@ func run(state overseer.State, logSync func() error) { printer = new(output.JSONPrinter) case *gitHubActionsFormat: printer = new(output.GitHubActionsPrinter) + case *sarifOut: + printer = new(output.SarifPrinter) default: printer = new(output.PlainPrinter) } - if !*jsonLegacy && !*jsonOut { + if !*jsonLegacy && !*jsonOut && !*sarifOut { fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n") } @@ -622,6 +689,7 @@ func run(state overseer.State, logSync func() error) { Dispatcher: engine.NewPrinterDispatcher(printer), FilterUnverified: *filterUnverified, FilterEntropy: *filterEntropy, + NoIgnoreTag: *noIgnoreTag, VerificationOverlap: *allowVerificationOverlap, Results: parsedResults, PrintAvgDetectorTime: *printAvgDetectorTime, @@ -656,6 +724,15 @@ func run(state overseer.State, logSync func() error) { logFatal(err, "error running scan") } + // SARIF can't be streamed like the other output formats: it's a single JSON document + // wrapping every result, so it's buffered by the printer and written out here, once + // scanning has fully finished. + if sarifPrinter, ok := printer.(*output.SarifPrinter); ok { + if err := sarifPrinter.Flush(os.Stdout); err != nil { + logFatal(err, "error writing SARIF output") + } + } + verificationCacheMetricsSnapshot := struct { Hits int32 Misses int32 @@ -753,7 +830,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, handleFinishedMetrics := func(ctx context.Context, finishedMetrics <-chan sources.UnitMetrics, jobReportWriter io.WriteCloser) { go func() { defer func() { - jobReportWriter.Close() + _ = jobReportWriter.Close() if namer, ok := jobReportWriter.(interface{ Name() string }); ok { ctx.Logger().Info("report written", "path", namer.Name()) } else { @@ -850,7 +927,11 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, // Override git configuration for pre-commit hook context gitCfg.TrustLocalGitConfig = true - gitCfg.BaseRef = "HEAD" // Only scan staged changes + // Only scan staged changes. Both ends are HEAD so the commit range is + // HEAD..HEAD, which is empty; a base with no head would instead walk + // every ref not reachable from HEAD (see gitparse.Parser.RepoPath). + gitCfg.BaseRef = "HEAD" + gitCfg.HeadRef = "HEAD" // Override result filters for pre-commit hook context // In hook mode, we only want to show verified secrets and unknown findings @@ -878,6 +959,13 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, if len(*githubScanOrgs) > 0 && len(*githubScanRepos) > 0 { return scanMetrics, fmt.Errorf("invalid config: you cannot specify both organizations and repositories at the same time") } + // --include-repos/--exclude-repos only filter repos enumerated from an org/user scan; + // explicit --repo entries bypass that filtering step entirely (see enumerate() in + // pkg/sources/github/github.go), so combining them with --repo would silently no-op + // the filters instead of erroring. + if len(*githubScanRepos) > 0 && (len(*githubIncludeRepos) > 0 || len(*githubExcludeRepos) > 0) { + return scanMetrics, fmt.Errorf("invalid config: --include-repos and --exclude-repos only apply to organization or user scans and cannot be used with --repo") + } if err := validateClonePath(*githubClonePath, *githubNoCleanup); err != nil { return scanMetrics, err @@ -903,6 +991,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, ClonePath: *githubClonePath, NoCleanup: *githubNoCleanup, IgnoreGists: *githubIgnoreGists, + ExcludeArchived: *githubExcludeArchived, PrintLegacyJSON: *jsonLegacy, } @@ -978,14 +1067,20 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, } case s3Scan.FullCommand(): cfg := sources.S3Config{ - Key: *s3ScanKey, - Secret: *s3ScanSecret, - SessionToken: *s3ScanSessionToken, - Buckets: *s3ScanBuckets, - IgnoreBuckets: *s3ScanIgnoreBuckets, - Roles: *s3ScanRoleArns, - CloudCred: *s3ScanCloudEnv, - MaxObjectSize: int64(*s3ScanMaxObjectSize), + Key: *s3ScanKey, + Secret: *s3ScanSecret, + SessionToken: *s3ScanSessionToken, + Buckets: *s3ScanBuckets, + IgnoreBuckets: *s3ScanIgnoreBuckets, + Roles: *s3ScanRoleArns, + CloudCred: *s3ScanCloudEnv, + MaxObjectSize: int64(*s3ScanMaxObjectSize), + Endpoint: *s3ScanEndpoint, + Region: *s3ScanRegion, + IncludePrefixes: *s3ScanIncludePrefixes, + ExcludePrefixes: *s3ScanExcludePrefixes, + IncludeExtensions: *s3ScanIncludeExtensions, + ExcludeExtensions: *s3ScanExcludeExtensions, } if ref, err := eng.ScanS3(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan S3: %v", err) @@ -1137,8 +1232,8 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, *huggingfaceEndpoint = strings.TrimRight(*huggingfaceEndpoint, "/") } - if len(*huggingfaceModels) == 0 && len(*huggingfaceSpaces) == 0 && len(*huggingfaceDatasets) == 0 && len(*huggingfaceOrgs) == 0 && len(*huggingfaceUsers) == 0 { - return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization, user, model, space or dataset") + if len(*huggingfaceModels) == 0 && len(*huggingfaceSpaces) == 0 && len(*huggingfaceDatasets) == 0 && len(*huggingfaceBuckets) == 0 && len(*huggingfaceOrgs) == 0 && len(*huggingfaceUsers) == 0 { + return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization, user, model, space, dataset or bucket") } cfg := engine.HuggingfaceConfig{ @@ -1146,18 +1241,22 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, Models: *huggingfaceModels, Spaces: *huggingfaceSpaces, Datasets: *huggingfaceDatasets, + Buckets: *huggingfaceBuckets, Organizations: *huggingfaceOrgs, Users: *huggingfaceUsers, Token: *huggingfaceToken, IncludeModels: *huggingfaceIncludeModels, IncludeSpaces: *huggingfaceIncludeSpaces, IncludeDatasets: *huggingfaceIncludeDatasets, + IncludeBuckets: *huggingfaceIncludeBuckets, IgnoreModels: *huggingfaceIgnoreModels, IgnoreSpaces: *huggingfaceIgnoreSpaces, IgnoreDatasets: *huggingfaceIgnoreDatasets, + IgnoreBuckets: *huggingfaceIgnoreBuckets, SkipAllModels: *huggingfaceSkipAllModels, SkipAllSpaces: *huggingfaceSkipAllSpaces, SkipAllDatasets: *huggingfaceSkipAllDatasets, + SkipAllBuckets: *huggingfaceSkipAllBuckets, IncludeDiscussions: *huggingfaceIncludeDiscussions, IncludePrs: *huggingfaceIncludePrs, Concurrency: *concurrency, diff --git a/pkg/analyzer/analyzers/airbrake/airbrake.go b/pkg/analyzer/analyzers/airbrake/airbrake.go index eba06658f2a0..494639e354b3 100644 --- a/pkg/analyzer/analyzers/airbrake/airbrake.go +++ b/pkg/analyzer/analyzers/airbrake/airbrake.go @@ -25,8 +25,7 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeAir func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { info, err := AnalyzePermissions(a.Cfg, credInfo["key"]) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -109,7 +108,7 @@ func validateKey(cfg *config.Config, key string) (bool, []Project, error) { } // read response - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // if status code is 200, decode response if resp.StatusCode == 200 { @@ -150,7 +149,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { return nil, err } if !valid { - return nil, fmt.Errorf("Invalid Airbrake User API Key") + return nil, fmt.Errorf("invalid Airbrake User API Key") } info := &SecretInfo{ diff --git a/pkg/analyzer/analyzers/airtable/airtablepat/airtable.go b/pkg/analyzer/analyzers/airtable/airtablepat/airtable.go index 2a87417991b5..97ca4f0fa8c5 100644 --- a/pkg/analyzer/analyzers/airtable/airtablepat/airtable.go +++ b/pkg/analyzer/analyzers/airtable/airtablepat/airtable.go @@ -119,7 +119,7 @@ func determineScope(token string, perm common.Permission, requiredIDs map[string if requiredIDs != nil { for _, key := range endpoint.RequiredIDs { if value, ok := requiredIDs[key]; ok { - url = strings.Replace(url, fmt.Sprintf("{%s}", key), value, -1) + url = strings.ReplaceAll(url, fmt.Sprintf("{%s}", key), value) } } } @@ -128,7 +128,7 @@ func determineScope(token string, perm common.Permission, requiredIDs map[string if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode == endpoint.ExpectedSuccessStatus { scopeStatusMap[scopeString] = true diff --git a/pkg/analyzer/analyzers/airtable/airtablepat/requests.go b/pkg/analyzer/analyzers/airtable/airtablepat/requests.go index 2afac070e646..88766832eea7 100644 --- a/pkg/analyzer/analyzers/airtable/airtablepat/requests.go +++ b/pkg/analyzer/analyzers/airtable/airtablepat/requests.go @@ -18,12 +18,12 @@ func fetchAirtableRecords(token string, baseID string, tableID string) ([]common if !exists { return nil, fmt.Errorf("endpoint for ListRecordsEndpoint does not exist") } - url := strings.Replace(strings.Replace(endpoint.URL, "{baseID}", baseID, -1), "{tableID}", tableID, -1) + url := strings.ReplaceAll(strings.ReplaceAll(endpoint.URL, "{baseID}", baseID), "{tableID}", tableID) resp, err := common.CallAirtableAPI(token, "GET", url) if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to fetch Airtable records, status: %d", resp.StatusCode) diff --git a/pkg/analyzer/analyzers/airtable/common/common.go b/pkg/analyzer/analyzers/airtable/common/common.go index cf989ca84fe0..6c9b7941d727 100644 --- a/pkg/analyzer/analyzers/airtable/common/common.go +++ b/pkg/analyzer/analyzers/airtable/common/common.go @@ -39,7 +39,7 @@ func FetchAirtableUserInfo(token string) (*AirtableUserInfo, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to fetch Airtable user info, status: %d", resp.StatusCode) @@ -62,7 +62,7 @@ func FetchAirtableBases(token string) (*AirtableBases, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to fetch Airtable bases, status: %d", resp.StatusCode) @@ -96,7 +96,7 @@ func fetchBaseSchema(token string, baseID string) (*Schema, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to fetch schema for base %s, status: %d", baseID, resp.StatusCode) diff --git a/pkg/analyzer/analyzers/analyzers.go b/pkg/analyzer/analyzers/analyzers.go index be2ad171de05..f2deaecb340f 100644 --- a/pkg/analyzer/analyzers/analyzers.go +++ b/pkg/analyzer/analyzers/analyzers.go @@ -211,7 +211,7 @@ func (h *HttpStatusTest) RunTest(headers map[string]string) error { } // Create new HTTP request - client := &http.Client{} + client := &http.Client{Transport: baseTransport()} req, err := http.NewRequest(h.Method, h.URL, data) if err != nil { return err @@ -227,7 +227,7 @@ func (h *HttpStatusTest) RunTest(headers map[string]string) error { if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { diff --git a/pkg/analyzer/analyzers/anthropic/anthropic.go b/pkg/analyzer/analyzers/anthropic/anthropic.go index 6b4b36b39a9d..e64314fd4b66 100644 --- a/pkg/analyzer/analyzers/anthropic/anthropic.go +++ b/pkg/analyzer/analyzers/anthropic/anthropic.go @@ -48,14 +48,16 @@ func (a Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerAnthropic } -func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { +func (a Analyzer) Analyze(ctx context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, exist := credInfo["key"] if !exist { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("key not found in credentials info"), + err := errors.New("key not found in credentials info") + ctx.Logger().Error(err, "anthropic credentials missing key") + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", err, ) } - secretInfo, err := AnalyzePermissions(a.Cfg, key) + secretInfo, err := AnalyzePermissions(ctx, a.Cfg, key) if err != nil { return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, ) @@ -65,7 +67,7 @@ func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analy } func AnalyzeAndPrintPermissions(cfg *config.Config, key string) { - info, err := AnalyzePermissions(cfg, key) + info, err := AnalyzePermissions(context.Background(), cfg, key) if err != nil { // just print the error in cli and continue as a partial success color.Red("[x] Error : %s", err.Error()) @@ -88,7 +90,7 @@ func AnalyzeAndPrintPermissions(cfg *config.Config, key string) { } } -func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { +func AnalyzePermissions(ctx context.Context, cfg *config.Config, key string) (*SecretInfo, error) { // create a HTTP client client := analyzers.NewAnalyzeClient(cfg) @@ -100,15 +102,17 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { switch keyType { case APIKey: - if err := captureAPIKeyResources(client, key, secretInfo); err != nil { + if err := captureAPIKeyResources(ctx, client, key, secretInfo); err != nil { return nil, err } case AdminKey: - if err := captureAdminKeyResources(client, key, secretInfo); err != nil { + if err := captureAdminKeyResources(ctx, client, key, secretInfo); err != nil { return nil, err } default: - return nil, errors.New("unsupported key type") + err := errors.New("unsupported key type") + ctx.Logger().Error(err, "unrecognized anthropic key format") + return nil, err } // anthropic key has full access only diff --git a/pkg/analyzer/analyzers/anthropic/requests.go b/pkg/analyzer/analyzers/anthropic/requests.go index 077f23d11450..622925b9075b 100644 --- a/pkg/analyzer/analyzers/anthropic/requests.go +++ b/pkg/analyzer/analyzers/anthropic/requests.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "net/http" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) var endpoints = map[string]string{ @@ -79,10 +81,11 @@ type APIKeysResponse struct { } // makeAnthropicRequest send the API request to passed url with passed key as API Key and return response body and status code -func makeAnthropicRequest(client *http.Client, url, key string) ([]byte, int, error) { +func makeAnthropicRequest(ctx context.Context, client *http.Client, url, key string) ([]byte, int, error) { // create request - req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { + ctx.Logger().Error(err, "failed to create anthropic request", "url", url) return nil, 0, err } @@ -93,6 +96,7 @@ func makeAnthropicRequest(client *http.Client, url, key string) ([]byte, int, er resp, err := client.Do(req) if err != nil { + ctx.Logger().Error(err, "failed to send anthropic request", "url", url) return nil, 0, err } @@ -103,6 +107,7 @@ func makeAnthropicRequest(client *http.Client, url, key string) ([]byte, int, er responseBodyByte, err := io.ReadAll(resp.Body) if err != nil { + ctx.Logger().Error(err, "failed to read anthropic response body", "url", url) return nil, 0, err } @@ -110,12 +115,12 @@ func makeAnthropicRequest(client *http.Client, url, key string) ([]byte, int, er } // captureAPIKeyResources capture resources associated with api key -func captureAPIKeyResources(client *http.Client, apiKey string, secretInfo *SecretInfo) error { - if err := captureModels(client, apiKey, secretInfo); err != nil { +func captureAPIKeyResources(ctx context.Context, client *http.Client, apiKey string, secretInfo *SecretInfo) error { + if err := captureModels(ctx, client, apiKey, secretInfo); err != nil { return err } - if err := captureMessageBatches(client, apiKey, secretInfo); err != nil { + if err := captureMessageBatches(ctx, client, apiKey, secretInfo); err != nil { return err } @@ -123,24 +128,24 @@ func captureAPIKeyResources(client *http.Client, apiKey string, secretInfo *Secr } // captureAdminKeyResources capture resources associated with admin key -func captureAdminKeyResources(client *http.Client, adminKey string, secretInfo *SecretInfo) error { - if err := captureOrgUsers(client, adminKey, secretInfo); err != nil { +func captureAdminKeyResources(ctx context.Context, client *http.Client, adminKey string, secretInfo *SecretInfo) error { + if err := captureOrgUsers(ctx, client, adminKey, secretInfo); err != nil { return err } - if err := captureWorkspaces(client, adminKey, secretInfo); err != nil { + if err := captureWorkspaces(ctx, client, adminKey, secretInfo); err != nil { return err } - if err := captureAPIKeys(client, adminKey, secretInfo); err != nil { + if err := captureAPIKeys(ctx, client, adminKey, secretInfo); err != nil { return err } return nil } -func captureModels(client *http.Client, apiKey string, secretInfo *SecretInfo) error { - response, statusCode, err := makeAnthropicRequest(client, endpoints["models"], apiKey) +func captureModels(ctx context.Context, client *http.Client, apiKey string, secretInfo *SecretInfo) error { + response, statusCode, err := makeAnthropicRequest(ctx, client, endpoints["models"], apiKey) if err != nil { return err } @@ -150,6 +155,7 @@ func captureModels(client *http.Client, apiKey string, secretInfo *SecretInfo) e var models ModelsResponse if err := json.Unmarshal(response, &models); err != nil { + ctx.Logger().Error(err, "failed to unmarshal models response") return err } @@ -163,14 +169,18 @@ func captureModels(client *http.Client, apiKey string, secretInfo *SecretInfo) e return nil case http.StatusNotFound, http.StatusUnauthorized: - return fmt.Errorf("invalid/revoked api-key") + err := fmt.Errorf("invalid/revoked api-key") + ctx.Logger().Error(err, "anthropic key invalid while fetching models", "status_code", statusCode) + return err default: - return fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + err := fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + ctx.Logger().Error(err, "unexpected status code while fetching models", "status_code", statusCode) + return err } } -func captureMessageBatches(client *http.Client, apiKey string, secretInfo *SecretInfo) error { - response, statusCode, err := makeAnthropicRequest(client, endpoints["messageBatches"], apiKey) +func captureMessageBatches(ctx context.Context, client *http.Client, apiKey string, secretInfo *SecretInfo) error { + response, statusCode, err := makeAnthropicRequest(ctx, client, endpoints["messageBatches"], apiKey) if err != nil { return err } @@ -180,6 +190,7 @@ func captureMessageBatches(client *http.Client, apiKey string, secretInfo *Secre var messageBatches MessageResponse if err := json.Unmarshal(response, &messageBatches); err != nil { + ctx.Logger().Error(err, "failed to unmarshal message batches response") return err } @@ -197,14 +208,18 @@ func captureMessageBatches(client *http.Client, apiKey string, secretInfo *Secre return nil case http.StatusNotFound, http.StatusUnauthorized: - return fmt.Errorf("invalid/revoked api-key") + err := fmt.Errorf("invalid/revoked api-key") + ctx.Logger().Error(err, "anthropic key invalid while fetching message batches", "status_code", statusCode) + return err default: - return fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + err := fmt.Errorf("unexpected status code: %d while fetching message batches", statusCode) + ctx.Logger().Error(err, "unexpected status code while fetching message batches", "status_code", statusCode) + return err } } -func captureOrgUsers(client *http.Client, adminKey string, secretInfo *SecretInfo) error { - response, statusCode, err := makeAnthropicRequest(client, endpoints["orgUsers"], adminKey) +func captureOrgUsers(ctx context.Context, client *http.Client, adminKey string, secretInfo *SecretInfo) error { + response, statusCode, err := makeAnthropicRequest(ctx, client, endpoints["orgUsers"], adminKey) if err != nil { return err } @@ -214,6 +229,7 @@ func captureOrgUsers(client *http.Client, adminKey string, secretInfo *SecretInf var users OrgUsersResponse if err := json.Unmarshal(response, &users); err != nil { + ctx.Logger().Error(err, "failed to unmarshal org users response") return err } @@ -231,14 +247,18 @@ func captureOrgUsers(client *http.Client, adminKey string, secretInfo *SecretInf return nil case http.StatusNotFound, http.StatusUnauthorized: - return fmt.Errorf("invalid/revoked api-key") + err := fmt.Errorf("invalid/revoked api-key") + ctx.Logger().Error(err, "anthropic key invalid while fetching org users", "status_code", statusCode) + return err default: - return fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + err := fmt.Errorf("unexpected status code: %d while fetching org users", statusCode) + ctx.Logger().Error(err, "unexpected status code while fetching org users", "status_code", statusCode) + return err } } -func captureWorkspaces(client *http.Client, adminKey string, secretInfo *SecretInfo) error { - response, statusCode, err := makeAnthropicRequest(client, endpoints["workspaces"], adminKey) +func captureWorkspaces(ctx context.Context, client *http.Client, adminKey string, secretInfo *SecretInfo) error { + response, statusCode, err := makeAnthropicRequest(ctx, client, endpoints["workspaces"], adminKey) if err != nil { return err } @@ -248,6 +268,7 @@ func captureWorkspaces(client *http.Client, adminKey string, secretInfo *SecretI var workspaces WorkspacesResponse if err := json.Unmarshal(response, &workspaces); err != nil { + ctx.Logger().Error(err, "failed to unmarshal workspaces response") return err } @@ -260,21 +281,25 @@ func captureWorkspaces(client *http.Client, adminKey string, secretInfo *SecretI secretInfo.AnthropicResources = append(secretInfo.AnthropicResources, resource) // capture each workspace members - if err := captureWorkspaceMembers(client, adminKey, resource, secretInfo); err != nil { + if err := captureWorkspaceMembers(ctx, client, adminKey, resource, secretInfo); err != nil { return err } } return nil case http.StatusNotFound, http.StatusUnauthorized: - return fmt.Errorf("invalid/revoked api-key") + err := fmt.Errorf("invalid/revoked api-key") + ctx.Logger().Error(err, "anthropic key invalid while fetching workspaces", "status_code", statusCode) + return err default: - return fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + err := fmt.Errorf("unexpected status code: %d while fetching workspaces", statusCode) + ctx.Logger().Error(err, "unexpected status code while fetching workspaces", "status_code", statusCode) + return err } } -func captureWorkspaceMembers(client *http.Client, key string, parentWorkspace AnthropicResource, secretInfo *SecretInfo) error { - response, statusCode, err := makeAnthropicRequest(client, fmt.Sprintf(endpoints["workspaceMembers"], parentWorkspace.ID), key) +func captureWorkspaceMembers(ctx context.Context, client *http.Client, key string, parentWorkspace AnthropicResource, secretInfo *SecretInfo) error { + response, statusCode, err := makeAnthropicRequest(ctx, client, fmt.Sprintf(endpoints["workspaceMembers"], parentWorkspace.ID), key) if err != nil { return err } @@ -284,6 +309,7 @@ func captureWorkspaceMembers(client *http.Client, key string, parentWorkspace An var members WorkspaceMembersResponse if err := json.Unmarshal(response, &members); err != nil { + ctx.Logger().Error(err, "failed to unmarshal workspace members response", "workspace_id", parentWorkspace.ID) return err } @@ -298,14 +324,18 @@ func captureWorkspaceMembers(client *http.Client, key string, parentWorkspace An return nil case http.StatusNotFound, http.StatusUnauthorized: - return fmt.Errorf("invalid/revoked api-key") + err := fmt.Errorf("invalid/revoked api-key") + ctx.Logger().Error(err, "anthropic key invalid while fetching workspace members", "workspace_id", parentWorkspace.ID, "status_code", statusCode) + return err default: - return fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + err := fmt.Errorf("unexpected status code: %d while fetching workspace members", statusCode) + ctx.Logger().Error(err, "unexpected status code while fetching workspace members", "workspace_id", parentWorkspace.ID, "status_code", statusCode) + return err } } -func captureAPIKeys(client *http.Client, adminKey string, secretInfo *SecretInfo) error { - response, statusCode, err := makeAnthropicRequest(client, endpoints["apiKeys"], adminKey) +func captureAPIKeys(ctx context.Context, client *http.Client, adminKey string, secretInfo *SecretInfo) error { + response, statusCode, err := makeAnthropicRequest(ctx, client, endpoints["apiKeys"], adminKey) if err != nil { return err } @@ -315,6 +345,7 @@ func captureAPIKeys(client *http.Client, adminKey string, secretInfo *SecretInfo var apiKeys APIKeysResponse if err := json.Unmarshal(response, &apiKeys); err != nil { + ctx.Logger().Error(err, "failed to unmarshal api keys response") return err } @@ -334,8 +365,12 @@ func captureAPIKeys(client *http.Client, adminKey string, secretInfo *SecretInfo return nil case http.StatusNotFound, http.StatusUnauthorized: - return fmt.Errorf("invalid/revoked api-key") + err := fmt.Errorf("invalid/revoked api-key") + ctx.Logger().Error(err, "anthropic key invalid while fetching api keys", "status_code", statusCode) + return err default: - return fmt.Errorf("unexpected status code: %d while fetching models", statusCode) + err := fmt.Errorf("unexpected status code: %d while fetching api keys", statusCode) + ctx.Logger().Error(err, "unexpected status code while fetching api keys", "status_code", statusCode) + return err } } diff --git a/pkg/analyzer/analyzers/asana/asana.go b/pkg/analyzer/analyzers/asana/asana.go index a25158ab282b..d3c277ce5278 100644 --- a/pkg/analyzer/analyzers/asana/asana.go +++ b/pkg/analyzer/analyzers/asana/asana.go @@ -116,10 +116,10 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { } if resp.StatusCode != 200 { - return nil, fmt.Errorf("Invalid Asana API Key") + return nil, fmt.Errorf("invalid Asana API Key") } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() err = json.NewDecoder(resp.Body).Decode(&me) if err != nil { @@ -127,7 +127,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { } if me.Data.Email == "" { - return nil, fmt.Errorf("Invalid Asana API Key") + return nil, fmt.Errorf("invalid Asana API Key") } return &me, nil } diff --git a/pkg/analyzer/analyzers/bitbucket/bitbucket.go b/pkg/analyzer/analyzers/bitbucket/bitbucket.go index fd8471998df3..59d0b211e466 100644 --- a/pkg/analyzer/analyzers/bitbucket/bitbucket.go +++ b/pkg/analyzer/analyzers/bitbucket/bitbucket.go @@ -146,7 +146,7 @@ func getScopesAndType(cfg *config.Config, key string) (string, []string, error) if err != nil { return "", nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // parse response headers credentialType := resp.Header.Get("x-credential-type") @@ -198,7 +198,7 @@ func getRepositories(cfg *config.Config, key string, role string) (RepoJSON, err if err != nil { return repos, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // parse response body err = json.NewDecoder(resp.Body).Decode(&repos) diff --git a/pkg/analyzer/analyzers/client.go b/pkg/analyzer/analyzers/client.go index 1bd9818676c8..269a030d312a 100644 --- a/pkg/analyzer/analyzers/client.go +++ b/pkg/analyzer/analyzers/client.go @@ -2,15 +2,76 @@ package analyzers import ( "fmt" + "net" "net/http" "os" "strings" + "sync/atomic" "time" "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config" + "github.com/trufflesecurity/trufflehog/v3/pkg/ssrf" "golang.org/x/time/rate" ) +// restrictEgress gates whether analyzer HTTP clients enforce the SSRF egress +// guard from pkg/ssrf. Analyzers take their endpoints from scanned content (a +// secret's domain, a connection string), so a hosted deployment must refuse +// dials into internal address space. Default false preserves behavior for the +// OSS CLI and self-hosted use, which legitimately analyze credentials for +// internal services. +var restrictEgress atomic.Bool + +// SetEgressRestriction enables or disables the analyzer SSRF egress guard. +// When enabled, every client built by this package (NewAnalyzeClient, +// NewAnalyzeClientUnrestricted, HttpStatusTest.RunTest, and the +// RateLimitRoundTripper fallback) refuses to connect to non-public addresses, +// checked after DNS resolution and re-checked on every redirect hop. +func SetEgressRestriction(enabled bool) { + restrictEgress.Store(enabled) +} + +// baseTransport returns the round tripper analyzer clients build on: the +// guarded transport when the egress restriction is enabled, otherwise +// http.DefaultTransport. +func baseTransport() http.RoundTripper { + if restrictEgress.Load() { + return safeEgressTransport + } + return http.DefaultTransport +} + +// safeEgressTransport is http.DefaultTransport with the sole modification of a +// guarded dialer (see ssrf.GuardDialer). Cloning preserves proxy and timeout +// settings; note the pkg/ssrf caveat that a forward proxy moves the final +// connection out of the dialer's sight, so egress policy must then also be +// enforced at the proxy. +var safeEgressTransport = newSafeEgressTransport() + +func newSafeEgressTransport() *http.Transport { + guarded, ok := http.DefaultTransport.(*http.Transport) + if ok { + guarded = guarded.Clone() + } else { + // http.DefaultTransport is always an *http.Transport; this is a + // defensive fallback mirroring the standard library's field values. + guarded = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + } + // Mirror http.DefaultTransport's dialer (30s timeout and keep-alive). + guarded.DialContext = ssrf.GuardDialer(&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext + return guarded +} + type AnalyzeClient struct { http.Client LoggingEnabled bool @@ -34,7 +95,7 @@ type ClientOption func(*http.Client) // This returns a client that is restricted and filters out unsafe requests returning a success status code. func NewAnalyzeClient(cfg *config.Config, opts ...func(*http.Client)) *http.Client { client := &http.Client{ - Transport: AnalyzerRoundTripper{parent: http.DefaultTransport}, + Transport: AnalyzerRoundTripper{parent: baseTransport()}, } if cfg != nil && cfg.LoggingEnabled { client = &http.Client{ @@ -53,7 +114,7 @@ func NewAnalyzeClient(cfg *config.Config, opts ...func(*http.Client)) *http.Clie // This returns a client that is unrestricted and does not filter out unsafe requests returning a success status code. func NewAnalyzeClientUnrestricted(cfg *config.Config, opts ...ClientOption) *http.Client { client := &http.Client{ - Transport: http.DefaultTransport, + Transport: baseTransport(), } if cfg != nil && cfg.LoggingEnabled { client = &http.Client{ @@ -107,7 +168,7 @@ func (r LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error if err != nil { return resp, fmt.Errorf("failed to open log file: %w", err) } - defer file.Close() + defer func() { _ = file.Close() }() // Write log entry to file. if _, err := file.WriteString(logEntry); err != nil { @@ -151,7 +212,7 @@ type RateLimitRoundTripper struct { func (rt RateLimitRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { if rt.parent == nil { - rt.parent = http.DefaultTransport + rt.parent = baseTransport() } if rt.limiter != nil { if err := rt.limiter.Wait(req.Context()); err != nil { diff --git a/pkg/analyzer/analyzers/datadog/requests.go b/pkg/analyzer/analyzers/datadog/requests.go index 9a9f7fd92b95..a4919607a694 100644 --- a/pkg/analyzer/analyzers/datadog/requests.go +++ b/pkg/analyzer/analyzers/datadog/requests.go @@ -216,7 +216,7 @@ func ValidateApiKey(client *http.Client, baseURL, apiKey string) (bool, error) { case http.StatusForbidden: return false, nil default: - return false, fmt.Errorf("Unable to validate api key with status code: %d", resp.StatusCode) + return false, fmt.Errorf("unable to validate api key with status code: %d", resp.StatusCode) } } diff --git a/pkg/analyzer/analyzers/digitalocean/digitalocean.go b/pkg/analyzer/analyzers/digitalocean/digitalocean.go index e2266deb9504..0b870046b0c1 100644 --- a/pkg/analyzer/analyzers/digitalocean/digitalocean.go +++ b/pkg/analyzer/analyzers/digitalocean/digitalocean.go @@ -34,13 +34,11 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeDig func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -125,7 +123,7 @@ func (h *HttpStatusTest) RunTest(cfg *config.Config, headers map[string]string) if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { @@ -232,7 +230,7 @@ func getUser(cfg *config.Config, token string) (*user, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/analyzer/analyzers/dockerhub/dockerhub.go b/pkg/analyzer/analyzers/dockerhub/dockerhub.go index 5584bd9e0cbf..bc400869d653 100644 --- a/pkg/analyzer/analyzers/dockerhub/dockerhub.go +++ b/pkg/analyzer/analyzers/dockerhub/dockerhub.go @@ -133,7 +133,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult { result := analyzers.AnalyzerResult{ AnalyzerType: analyzers.AnalyzerTypeDockerHub, Metadata: map[string]any{"Valid_Key": info.Valid}, - Bindings: make([]analyzers.Binding, len(info.Repositories)), + Bindings: make([]analyzers.Binding, 0, len(info.Repositories)), } // extract information to create bindings and append to result bindings diff --git a/pkg/analyzer/analyzers/dockerhub/dockerhub_test.go b/pkg/analyzer/analyzers/dockerhub/dockerhub_test.go index 6cc37749c0ef..512ab6c36e40 100644 --- a/pkg/analyzer/analyzers/dockerhub/dockerhub_test.go +++ b/pkg/analyzer/analyzers/dockerhub/dockerhub_test.go @@ -15,6 +15,90 @@ import ( //go:embed result_output.json var expectedOutput []byte +func repo(name string) Repository { + return Repository{ + ID: "user123/repo/image/" + name, + Name: name, + Type: "image", + IsPrivate: true, + StarCount: 1, + PullCount: 2, + } +} + +func TestSecretInfoToAnalyzerResult(t *testing.T) { + tests := []struct { + name string + repos []Repository + wantLen int + wantNames []string + }{ + { + name: "one repository produces exactly one binding", + repos: []Repository{repo("repo1")}, + wantLen: 1, + wantNames: []string{"repo1"}, + }, + { + name: "three repositories produce exactly three bindings, not six", + repos: []Repository{repo("repo1"), repo("repo2"), repo("repo3")}, + wantLen: 3, + wantNames: []string{"repo1", "repo2", "repo3"}, + }, + { + name: "no repositories: no bindings", + repos: nil, + wantLen: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := &SecretInfo{ + Valid: true, + User: User{ID: "uuid-123", Username: "user123", Email: "user123@example.com"}, + Permissions: []string{"repo:admin"}, + Repositories: tt.repos, + } + + result := secretInfoToAnalyzerResult(info) + if result == nil { + t.Fatal("secretInfoToAnalyzerResult() returned nil") + } + + if len(result.Bindings) != tt.wantLen { + t.Errorf("got %d bindings, want %d", len(result.Bindings), tt.wantLen) + } + + // Mirrors the four proto min_len constraints so this test fails for + // the same reason the enterprise API does. + for i, binding := range result.Bindings { + if binding.Resource.Name == "" { + t.Errorf("binding[%d].Resource.Name is empty", i) + } + if binding.Resource.Type == "" { + t.Errorf("binding[%d].Resource.Type is empty", i) + } + if binding.Resource.FullyQualifiedName == "" { + t.Errorf("binding[%d].Resource.FullyQualifiedName is empty", i) + } + if binding.Permission.Value == "" { + t.Errorf("binding[%d].Permission.Value is empty", i) + } + } + + for i, name := range tt.wantNames { + if i >= len(result.Bindings) { + break + } + if result.Bindings[i].Resource.Name != name { + t.Errorf("binding[%d].Resource.Name = %q, want %q", i, result.Bindings[i].Resource.Name, name) + } + } + }) + } +} + func TestAnalyzer_Analyze(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() diff --git a/pkg/analyzer/analyzers/elevenlabs/elevenlabs.go b/pkg/analyzer/analyzers/elevenlabs/elevenlabs.go index 7cb07b578fca..1234a092cb5c 100644 --- a/pkg/analyzer/analyzers/elevenlabs/elevenlabs.go +++ b/pkg/analyzer/analyzers/elevenlabs/elevenlabs.go @@ -86,14 +86,12 @@ func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analy // check if the `key` exist in the credentials info key, exist := credInfo["key"] if !exist { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("key not found in credentials info"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("key not found in credentials info")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil @@ -242,9 +240,10 @@ func fetchUser(client *http.Client, key string) (*User, error) { return nil, err } - if errorResp.Detail.Status == InvalidAPIKey || errorResp.Detail.Status == NotVerifiable { + switch errorResp.Detail.Status { + case InvalidAPIKey, NotVerifiable: return nil, errors.New("invalid api key") - } else if errorResp.Detail.Status == MissingPermissions { + case MissingPermissions: // key is missing user read permissions but is valid return nil, nil } diff --git a/pkg/analyzer/analyzers/figma/figma.go b/pkg/analyzer/analyzers/figma/figma.go index fc381bedebe2..c00a36cae2cb 100644 --- a/pkg/analyzer/analyzers/figma/figma.go +++ b/pkg/analyzer/analyzers/figma/figma.go @@ -7,10 +7,10 @@ import ( "encoding/json" "errors" "fmt" + regexp "github.com/wasilibs/go-re2" "io" "net/http" "os" - "regexp" "strings" "github.com/fatih/color" @@ -83,7 +83,7 @@ func AnalyzePermissions(cfg *config.Config, token string) (*secretInfo, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err diff --git a/pkg/analyzer/analyzers/github/classic/classictoken.go b/pkg/analyzer/analyzers/github/classic/classictoken.go index 72b451cccf3a..6afcc5a8e7d7 100644 --- a/pkg/analyzer/analyzers/github/classic/classictoken.go +++ b/pkg/analyzer/analyzers/github/classic/classictoken.go @@ -95,9 +95,11 @@ func AnalyzeClassicToken(client *gh.Client, meta *common.TokenMetadata) (*common } } + // Gist enumeration is supplemental -- a rate-limit or transient failure + // should not discard the already-gathered metadata, repos, and permissions. gists, err := common.GetAllGistsForUser(client) if err != nil { - return nil, fmt.Errorf("enumerating gists for classic PAT: %w", err) + gists = nil } return &common.SecretInfo{ diff --git a/pkg/analyzer/analyzers/github/finegrained/finegrained.go b/pkg/analyzer/analyzers/github/finegrained/finegrained.go index d796a3001779..43289f1b7d70 100644 --- a/pkg/analyzer/analyzers/github/finegrained/finegrained.go +++ b/pkg/analyzer/analyzers/github/finegrained/finegrained.go @@ -78,7 +78,7 @@ func permissionFormatter(key, val any) (string, string) { if perm, ok := val.(Permission); ok { permStr, err := perm.ToString() if err != nil { - log.Fatal(fmt.Errorf("Error converting permission to string: %v", err)) + log.Fatal(fmt.Errorf("error converting permission to string: %v", err)) } var permissionStr string switch { @@ -119,7 +119,7 @@ func getMetadataPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> GET request to /repos/{owner}/{repo}/collaborators _, resp, err := client.Repositories.ListCollaborators(context.Background(), *repo.Owner.Login, *repo.Name, nil) if err != nil { - if resp.StatusCode == 403 { + if resp != nil && resp.StatusCode == 403 { return NoAccess, nil } return Invalid, err @@ -134,6 +134,9 @@ func getActionsPermission(client *gh.Client, repo *gh.Repository, currentAccess // Risk: Extremely Low // -> GET request to /repos/{owner}/{repo}/actions/artifacts _, resp, err := client.Actions.ListArtifacts(context.Background(), *repo.Owner.Login, *repo.Name, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -147,6 +150,9 @@ func getActionsPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> Unless the user has a workflow file named (see RANDOM_STRING above), this will always return 404 for users with READ_WRITE permissions. // -> POST request to /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches resp, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, gh.CreateWorkflowDispatchEventRequest{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return ActionsRead, nil @@ -167,6 +173,9 @@ func getActionsPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> Unless the user has a workflow file named (see RANDOM_STRING above), this will always return 404 for users with READ_WRITE permissions. // -> POST request to /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches resp, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, gh.CreateWorkflowDispatchEventRequest{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -187,6 +196,9 @@ func getAdministrationPermission(client *gh.Client, repo *gh.Repository, current // Risk: Extremely Low // -> GET request to /repos/{owner}/{repo}/actions/permissions _, resp, err := client.Repositories.GetActionsPermissions(context.Background(), *repo.Owner.Login, *repo.Name) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -203,6 +215,9 @@ func getAdministrationPermission(client *gh.Client, repo *gh.Repository, current return Invalid, err } resp, err = client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return AdministrationRead, nil @@ -217,7 +232,10 @@ func getCodeScanningAlertsPermission(client *gh.Client, repo *gh.Repository, cur // Risk: Extremely Low // -> GET request to /repos/{owner}/{repo}/code-scanning/alerts _, resp, err := client.CodeScanning.ListAlertsForRepo(context.Background(), *repo.Owner.Login, *repo.Name, nil) - defer resp.Body.Close() + if resp == nil { + return Invalid, err + } + defer func() { _ = resp.Body.Close() }() switch { case resp.StatusCode == 403: @@ -234,6 +252,9 @@ func getCodeScanningAlertsPermission(client *gh.Client, repo *gh.Repository, cur // -> Even if user had an alert with the number (see RANDOM_INTEGER above), this should error 422 due to the nil value passed in. // -> PATCH request to /repos/{owner}/{repo}/code-scanning/alerts/{alert_number} _, resp, err = client.CodeScanning.UpdateAlert(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_INTEGER, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return CodeScanningAlertsRead, nil @@ -251,6 +272,9 @@ func getCodespacesPermission(client *gh.Client, repo *gh.Repository, currentAcce // Risk: Extremely Low // GET request to /repos/{owner}/{repo}/codespaces _, resp, err := client.Codespaces.ListInRepo(context.Background(), *repo.Owner.Login, *repo.Name, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -267,6 +291,9 @@ func getCodespacesPermission(client *gh.Client, repo *gh.Repository, currentAcce return Invalid, err } resp, err = client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return CodespacesRead, nil @@ -287,6 +314,9 @@ func getCodespacesMetadataPermission(client *gh.Client, repo *gh.Repository, cur return Invalid, err } resp, err := client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -301,6 +331,9 @@ func getCodespacesSecretsPermission(client *gh.Client, repo *gh.Repository, curr // Risk: Extremely Low // GET request to /repos/{owner}/{repo}/codespaces/secrets for non-existent secret _, resp, err := client.Codespaces.GetRepoSecret(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -322,6 +355,9 @@ func getCommitStatusesPermission(client *gh.Client, repo *gh.Repository, current // Risk: Extremely Low // GET request to /repos/{owner}/{repo}/commits/{commit_sha}/statuses _, resp, err := client.Repositories.ListStatuses(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -336,6 +372,9 @@ func getCommitStatusesPermission(client *gh.Client, repo *gh.Repository, current // -> We're POSTing a commit status to a commit that cannot exist. This should always return 422 if valid access. // POST request to /repos/{owner}/{repo}/statuses/{commit_sha} _, resp, err = client.Repositories.CreateStatus(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, &gh.RepoStatus{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return CommitStatusesRead, nil @@ -353,6 +392,9 @@ func getCommitStatusesPermission(client *gh.Client, repo *gh.Repository, current // -> We're POSTing a commit status to a commit that cannot exist. This should always return 422 if valid access. // POST request to /repos/{owner}/{repo}/statuses/{commit_sha} _, resp, err := client.Repositories.CreateStatus(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, &gh.RepoStatus{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: // All we know is we don't have READ_WRITE @@ -374,6 +416,9 @@ func getContentsPermission(client *gh.Client, repo *gh.Repository, currentAccess // Risk: Extremely Low // GET request to /repos/{owner}/{repo}/commits _, resp, err := client.Repositories.ListCommits(context.Background(), *repo.Owner.Login, *repo.Name, &gh.CommitsListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -390,6 +435,9 @@ func getContentsPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> We're creating a file with an invalid payload. Worst case is a file with a random string and no content is created. But this should never happen. // PUT /repos/{owner}/{repo}/contents/{path} _, resp, err = client.Repositories.CreateFile(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, &gh.RepositoryContentFileOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return ContentsRead, nil @@ -410,6 +458,9 @@ func getContentsPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> We're creating a file with an invalid payload. Worst case is a file with a random string and no content is created. But this should never happen. // PUT /repos/{owner}/{repo}/contents/{path} _, resp, err := client.Repositories.CreateFile(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, &gh.RepositoryContentFileOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -427,7 +478,10 @@ func getDependabotAlertsPermission(client *gh.Client, repo *gh.Repository, curre // Risk: Extremely Low // GET /repos/{owner}/{repo}/dependabot/alerts _, resp, err := client.Dependabot.ListRepoAlerts(context.Background(), *repo.Owner.Login, *repo.Name, &gh.ListAlertsOptions{}) - defer resp.Body.Close() + if resp == nil { + return Invalid, err + } + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case 403: @@ -440,6 +494,9 @@ func getDependabotAlertsPermission(client *gh.Client, repo *gh.Repository, curre // PATCH /repos/{owner}/{repo}/dependabot/alerts/{alert_number} _, resp, err = client.Dependabot.UpdateAlert(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_INTEGER, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return DependabotAlertsRead, nil @@ -457,6 +514,9 @@ func getDependabotSecretsPermission(client *gh.Client, repo *gh.Repository, curr // Risk: Extremely Low // GET /repos/{owner}/{repo}/dependabot/secrets _, resp, err := client.Dependabot.ListRepoSecrets(context.Background(), *repo.Owner.Login, *repo.Name, &gh.ListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -470,6 +530,9 @@ func getDependabotSecretsPermission(client *gh.Client, repo *gh.Repository, curr // -> We're "creating" a secret with an invalid payload. Even if we did, the name would be (see RANDOM_STRING above) and the value would be nil. // PUT /repos/{owner}/{repo}/dependabot/secrets/{secret_name} resp, err = client.Dependabot.CreateOrUpdateRepoSecret(context.Background(), *repo.Owner.Login, *repo.Name, &gh.DependabotEncryptedSecret{Name: RANDOM_STRING}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return DependabotSecretsRead, nil @@ -487,6 +550,9 @@ func getDeploymentsPermission(client *gh.Client, repo *gh.Repository, currentAcc // Risk: Extremely Low // GET /repos/{owner}/{repo}/deployments _, resp, err := client.Repositories.ListDeployments(context.Background(), *repo.Owner.Login, *repo.Name, &gh.DeploymentsListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -500,6 +566,9 @@ func getDeploymentsPermission(client *gh.Client, repo *gh.Repository, currentAcc // -> We're creating a deployment with an invalid payload. Even if we did, the name would be (see RANDOM_STRING above) and the value would be nil. // POST /repos/{owner}/{repo}/deployments/{deployment_id}/statuses _, resp, err = client.Repositories.CreateDeployment(context.Background(), *repo.Owner.Login, *repo.Name, &gh.DeploymentRequest{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return DeploymentsRead, nil @@ -517,7 +586,7 @@ func getEnvironmentsPermission(client *gh.Client, repo *gh.Repository, currentAc // Risk: Extremely Low // GET /repos/{owner}/{repo}/environments envResp, resp, _ := client.Repositories.ListEnvironments(context.Background(), *repo.Owner.Login, *repo.Name, &gh.EnvironmentListOptions{}) - if resp.StatusCode != 200 { + if resp == nil || resp.StatusCode != 200 { return NoAccess, nil } // If no environments exist, then we return UNKNOWN @@ -528,6 +597,9 @@ func getEnvironmentsPermission(client *gh.Client, repo *gh.Repository, currentAc // Risk: Extremely Low // GET /repositories/{repository_id}/environments/{environment_name}/variables _, resp, _ = client.Actions.ListEnvVariables(context.Background(), *repo.Owner.Login, *repo.Name, *envResp.Environments[0].Name, &gh.ListOptions{}) + if resp == nil { + return Invalid, nil + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -541,6 +613,9 @@ func getEnvironmentsPermission(client *gh.Client, repo *gh.Repository, currentAc // -> We're updating an environment variable with an invalid payload. Even if we did, the name would be (see RANDOM_STRING above) and the value would be nil. // PATCH /repositories/{repository_id}/environments/{environment_name}/variables/{variable_name} resp, err := client.Actions.UpdateEnvVariable(context.Background(), *repo.Owner.Login, *repo.Name, *envResp.Environments[0].Name, &gh.ActionsVariable{Name: RANDOM_STRING}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return EnvironmentsRead, nil @@ -561,6 +636,9 @@ func getIssuesPermission(client *gh.Client, repo *gh.Repository, currentAccess P // Risk: Extremely Low // GET /repos/{owner}/{repo}/issues _, resp, err := client.Issues.ListByRepo(context.Background(), *repo.Owner.Login, *repo.Name, &gh.IssueListByRepoOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -574,6 +652,9 @@ func getIssuesPermission(client *gh.Client, repo *gh.Repository, currentAccess P // -> We're editing an issue label that does not exist. Even if we did, the name would be (see RANDOM_STRING above). // PATCH /repos/{owner}/{repo}/labels/{name} _, resp, err = client.Issues.EditLabel(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, &gh.Label{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return IssuesRead, nil @@ -594,6 +675,9 @@ func getIssuesPermission(client *gh.Client, repo *gh.Repository, currentAccess P // -> We're editing an issue label that does not exist. Even if we did, the name would be (see RANDOM_STRING above). // PATCH /repos/{owner}/{repo}/labels/{name} _, resp, err := client.Issues.EditLabel(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_STRING, &gh.Label{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -613,6 +697,9 @@ func getPagesPermission(client *gh.Client, repo *gh.Repository, currentAccess Pe // Risk: Extremely Low // GET /repos/{owner}/{repo}/pages _, resp, err := client.Repositories.GetPagesInfo(context.Background(), *repo.Owner.Login, *repo.Name) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -630,6 +717,9 @@ func getPagesPermission(client *gh.Client, repo *gh.Repository, currentAccess Pe return Invalid, err } resp, err = client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return PagesRead, nil @@ -654,6 +744,9 @@ func getPagesPermission(client *gh.Client, repo *gh.Repository, currentAccess Pe return Invalid, err } resp, err := client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -673,6 +766,9 @@ func getPullRequestsPermission(client *gh.Client, repo *gh.Repository, currentAc // Risk: Extremely Low // GET /repos/{owner}/{repo}/pulls _, resp, err := client.PullRequests.List(context.Background(), *repo.Owner.Login, *repo.Name, &gh.PullRequestListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -686,6 +782,9 @@ func getPullRequestsPermission(client *gh.Client, repo *gh.Repository, currentAc // -> We're creating a pull request with an invalid payload. // POST /repos/{owner}/{repo}/pulls _, resp, err = client.PullRequests.Create(context.Background(), *repo.Owner.Login, *repo.Name, &gh.NewPullRequest{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return PullRequestsRead, nil @@ -706,6 +805,9 @@ func getPullRequestsPermission(client *gh.Client, repo *gh.Repository, currentAc // -> We're creating a pull request with an invalid payload. // POST /repos/{owner}/{repo}/pulls _, resp, err := client.PullRequests.Create(context.Background(), *repo.Owner.Login, *repo.Name, &gh.NewPullRequest{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -726,6 +828,9 @@ func getRepoSecurityPermission(client *gh.Client, repo *gh.Repository, currentAc // Risk: Extremely Low // GET /repos/{owner}/{repo}/security-advisories _, resp, err := client.SecurityAdvisories.ListRepositorySecurityAdvisories(context.Background(), *repo.Owner.Login, *repo.Name, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -743,6 +848,9 @@ func getRepoSecurityPermission(client *gh.Client, repo *gh.Repository, currentAc return Invalid, err } resp, err = client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return RepoSecurityRead, nil @@ -767,6 +875,9 @@ func getRepoSecurityPermission(client *gh.Client, repo *gh.Repository, currentAc return Invalid, err } resp, err := client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -785,6 +896,9 @@ func getSecretScanningPermission(client *gh.Client, repo *gh.Repository, current // Risk: Extremely Low // GET /repos/{owner}/{repo}/secret-scanning/alerts _, resp, err := client.SecretScanning.ListAlertsForRepo(context.Background(), *repo.Owner.Login, *repo.Name, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -798,6 +912,9 @@ func getSecretScanningPermission(client *gh.Client, repo *gh.Repository, current // -> We're updating a secret scanning alert for an alert that doesn't exist. // POST /repos/{owner}/{repo}/secret-scanning/alerts _, resp, err = client.SecretScanning.UpdateAlert(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_INTEGER, &gh.SecretScanningAlertUpdateOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return SecretScanningRead, nil @@ -815,6 +932,9 @@ func getSecretsPermission(client *gh.Client, repo *gh.Repository, currentAccess // Risk: Extremely Low // GET /repos/{owner}/{repo}/actions/secrets _, resp, err := client.Actions.ListRepoSecrets(context.Background(), *repo.Owner.Login, *repo.Name, &gh.ListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -828,6 +948,9 @@ func getSecretsPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> We're creating a secret with an invalid payload. // PUT /repos/{owner}/{repo}/actions/secrets/{secret_name} resp, err = client.Actions.CreateOrUpdateRepoSecret(context.Background(), *repo.Owner.Login, *repo.Name, &gh.EncryptedSecret{Name: RANDOM_STRING}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return SecretsRead, nil @@ -845,6 +968,9 @@ func getVariablesPermission(client *gh.Client, repo *gh.Repository, currentAcces // Risk: Extremely Low // GET /repos/{owner}/{repo}/actions/variables _, resp, err := client.Actions.ListRepoVariables(context.Background(), *repo.Owner.Login, *repo.Name, &gh.ListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -858,6 +984,9 @@ func getVariablesPermission(client *gh.Client, repo *gh.Repository, currentAcces // -> We're updating a variable that doesn't exist with an invalid payload. // PATCH /repos/{owner}/{repo}/actions/variables/{name} resp, err = client.Actions.UpdateRepoVariable(context.Background(), *repo.Owner.Login, *repo.Name, &gh.ActionsVariable{Name: RANDOM_STRING}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return VariablesRead, nil @@ -875,6 +1004,9 @@ func getWebhooksPermission(client *gh.Client, repo *gh.Repository, currentAccess // Risk: Extremely Low // GET /repos/{owner}/{repo}/hooks _, resp, err := client.Repositories.ListHooks(context.Background(), *repo.Owner.Login, *repo.Name, &gh.ListOptions{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -888,6 +1020,9 @@ func getWebhooksPermission(client *gh.Client, repo *gh.Repository, currentAccess // -> We're updating a webhook that doesn't exist with an invalid payload. // PATCH /repos/{owner}/{repo}/hooks/{hook_id} _, resp, err = client.Repositories.EditHook(context.Background(), *repo.Owner.Login, *repo.Name, RANDOM_INTEGER, &gh.Hook{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return WebhooksRead, nil @@ -926,6 +1061,9 @@ func getBlockUserPermission(client *gh.Client, user *gh.User) (Permission, error // Risk: Extremely Low // -> GET request to /user/blocks _, resp, err := client.Users.ListBlockedUsers(context.Background(), nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -939,6 +1077,9 @@ func getBlockUserPermission(client *gh.Client, user *gh.User) (Permission, error // -> PUT request to /user/blocks/{username} // -> We're blocking a user that doesn't exist. See RANDOM_STRING above. resp, err = client.Users.BlockUser(context.Background(), RANDOM_STRING) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return BlockUserRead, nil @@ -956,6 +1097,9 @@ func getCodespacesUserPermission(client *gh.Client, user *gh.User) (Permission, // Risk: Extremely Low // GET request to /user/codespaces/secrets _, resp, err := client.Codespaces.ListUserSecrets(context.Background(), nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -969,6 +1113,9 @@ func getCodespacesUserPermission(client *gh.Client, user *gh.User) (Permission, // PUT request to /user/codespaces/secrets/{secret_name} // Payload is invalid, so it shouldn't actually post. resp, err = client.Codespaces.CreateOrUpdateUserSecret(context.Background(), &gh.EncryptedSecret{Name: RANDOM_STRING}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return CodespaceUserSecretsRead, nil @@ -986,6 +1133,9 @@ func getEmailPermission(client *gh.Client, user *gh.User) (Permission, error) { // Risk: Extremely Low // GET request to /user/emails _, resp, err := client.Users.ListEmails(context.Background(), nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -998,6 +1148,9 @@ func getEmailPermission(client *gh.Client, user *gh.User) (Permission, error) { // Risk: Low // POST request to /user/emails/visibility _, resp, err = client.Users.SetEmailVisibility(context.Background(), RANDOM_STRING) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return EmailRead, nil @@ -1015,6 +1168,9 @@ func getFollowersPermission(client *gh.Client, user *gh.User) (Permission, error // Risk: Extremely Low // GET request to /user/followers _, resp, err := client.Users.ListFollowers(context.Background(), "", nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -1030,6 +1186,9 @@ func getFollowersPermission(client *gh.Client, user *gh.User) (Permission, error // an account for RANDOM_USERNAME value will then no longer follow that account. // But we're using an account created specifically for this purpose with no activity. resp, err = client.Users.Unfollow(context.Background(), RANDOM_USERNAME) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return FollowersRead, nil @@ -1044,6 +1203,9 @@ func getGPGKeysPermission(client *gh.Client, user *gh.User) (Permission, error) // Risk: Extremely Low // GET request to /user/gpg_keys _, resp, err := client.Users.ListGPGKeys(context.Background(), "", nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -1057,6 +1219,9 @@ func getGPGKeysPermission(client *gh.Client, user *gh.User) (Permission, error) // POST request to /user/gpg_keys // Payload is invalid, so it shouldn't actually post. _, resp, err = client.Users.CreateGPGKey(context.Background(), RANDOM_STRING) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return GpgKeysRead, nil @@ -1075,6 +1240,9 @@ func getGistsPermission(client *gh.Client, user *gh.User) (Permission, error) { // POST request to /gists // Payload is invalid, so it shouldn't actually post. _, resp, err := client.Gists.Create(context.Background(), &gh.Gist{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -1092,6 +1260,9 @@ func getGitKeysPermission(client *gh.Client, user *gh.User) (Permission, error) // Risk: Extremely Low // GET request to /user/keys _, resp, err := client.Users.ListKeys(context.Background(), "", nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -1105,6 +1276,9 @@ func getGitKeysPermission(client *gh.Client, user *gh.User) (Permission, error) // POST request to /user/keys // Payload is invalid, so it shouldn't actually post. _, resp, err = client.Users.CreateKey(context.Background(), &gh.Key{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return GitKeysRead, nil @@ -1126,6 +1300,9 @@ func getLimitsPermission(client *gh.Client, user *gh.User) (Permission, error) { return Invalid, err } resp, err := client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return NoAccess, nil @@ -1143,6 +1320,9 @@ func getLimitsPermission(client *gh.Client, user *gh.User) (Permission, error) { return Invalid, err } resp, err = client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return LimitsRead, nil @@ -1160,6 +1340,9 @@ func getPlanPermission(client *gh.Client, user *gh.User) (Permission, error) { // Risk: Extremely Low // GET request to /user/{username}/settings/billing/actions _, resp, err := client.Billing.GetActionsBillingUser(context.Background(), *user.Login) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -1179,6 +1362,9 @@ func getProfilePermission(client *gh.Client, user *gh.User) (Permission, error) return Invalid, err } resp, err := client.Do(context.Background(), req, nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -1196,6 +1382,9 @@ func getSigningKeysPermission(client *gh.Client, user *gh.User) (Permission, err // Risk: Extremely Low // GET request to /user/ssh_signing_keys _, resp, err := client.Users.ListSSHSigningKeys(context.Background(), "", nil) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403, 404: return NoAccess, nil @@ -1209,6 +1398,9 @@ func getSigningKeysPermission(client *gh.Client, user *gh.User) (Permission, err // POST request to /user/ssh_signing_keys // Payload is invalid, so it shouldn't actually post. _, resp, err = client.Users.CreateSSHSigningKey(context.Background(), &gh.Key{}) + if resp == nil { + return Invalid, err + } switch resp.StatusCode { case 403: return SigningKeysRead, nil @@ -1255,9 +1447,11 @@ func AnalyzeFineGrainedToken(client *gh.Client, meta *common.TokenMetadata, shal return nil, err } + // Gist enumeration is supplemental -- a rate-limit or transient failure + // should not discard the already-gathered metadata, repos, and permissions. allGists, err := common.GetAllGistsForUser(client) if err != nil { - return nil, err + allGists = nil } accessibleRepos := make([]*gh.Repository, 0) for _, repo := range allRepos { diff --git a/pkg/analyzer/analyzers/github/github.go b/pkg/analyzer/analyzers/github/github.go index 360123871ce4..32954582d5eb 100644 --- a/pkg/analyzer/analyzers/github/github.go +++ b/pkg/analyzer/analyzers/github/github.go @@ -50,9 +50,10 @@ func secretInfoToAnalyzerResult(info *common.SecretInfo) *analyzers.AnalyzerResu result := &analyzers.AnalyzerResult{ AnalyzerType: analyzers.AnalyzerTypeGitHub, Metadata: map[string]any{ - "owner": info.Metadata.User.Login, - "type": info.Metadata.Type, - "expiration": info.Metadata.Expiration, + "owner": info.Metadata.User.Login, + "type": info.Metadata.Type, + "expiration": info.Metadata.Expiration, + "is_fine_grained": info.Metadata.FineGrained, }, } result.Bindings = append(result.Bindings, secretInfoToUserBindings(info)...) @@ -120,8 +121,17 @@ func secretInfoToRepoBindings(info *common.SecretInfo) []analyzers.Binding { if owner := repo.GetOwner(); owner != nil { parent = userToResource(owner) } + // repo.GetName() could theoretically be empty for a deleted/ghost repo + // object; fall back to FullName so Name is never empty. + name := repo.GetName() + if name == "" { + name = repo.GetFullName() + } + if name == "" { + continue + } resource := analyzers.Resource{ - Name: repo.GetName(), + Name: name, FullyQualifiedName: fmt.Sprintf("github.com/%s", repo.GetFullName()), Type: "repository", Parent: parent, @@ -140,8 +150,14 @@ func secretInfoToGistBindings(info *common.SecretInfo) []analyzers.Binding { if gist.GetOwner() == nil { continue } + // Gist descriptions are optional; fall back to the gist ID so Name is + // never empty (proto validation requires len >= 1). + name := gist.GetDescription() + if name == "" { + name = gist.GetID() + } resource := analyzers.Resource{ - Name: gist.GetDescription(), + Name: name, FullyQualifiedName: fmt.Sprintf("gist.github.com/%s/%s", gist.GetOwner().GetLogin(), gist.GetID()), Type: "gist", Parent: userToResource(gist.GetOwner()), diff --git a/pkg/analyzer/analyzers/github/github_test.go b/pkg/analyzer/analyzers/github/github_test.go index 6ac3d3b6532e..2016b42343ee 100644 --- a/pkg/analyzer/analyzers/github/github_test.go +++ b/pkg/analyzer/analyzers/github/github_test.go @@ -20,7 +20,9 @@ func makeOwner(login, userType string) *gh.User { } // TestSecretInfoToGistBindings exercises nil-field edge cases that caused a -// production panic (nil Description and nil Owner from the GitHub API). +// production panic (nil Description and nil Owner from the GitHub API), and +// the empty-description proto-validation failure (CSM-1554) where Name must +// be non-empty. func TestSecretInfoToGistBindings(t *testing.T) { owner := makeOwner("truffle-sandbox", "User") scope := []analyzers.Permission{{Value: "gist"}} @@ -33,13 +35,22 @@ func TestSecretInfoToGistBindings(t *testing.T) { wantNames []string // parallel to wantFQNs; checked unconditionally (including empty string) }{ { - name: "nil description produces empty name, gist is still included", + name: "nil description falls back to gist ID as name", gists: []*gh.Gist{ {ID: strPtr("abc123"), Description: nil, Owner: owner}, }, wantLen: 1, wantFQNs: []string{"gist.github.com/truffle-sandbox/abc123"}, - wantNames: []string{""}, + wantNames: []string{"abc123"}, + }, + { + name: "empty description falls back to gist ID as name", + gists: []*gh.Gist{ + {ID: strPtr("abc123"), Description: strPtr(""), Owner: owner}, + }, + wantLen: 1, + wantFQNs: []string{"gist.github.com/truffle-sandbox/abc123"}, + wantNames: []string{"abc123"}, }, { name: "nil owner: gist is skipped entirely", @@ -107,7 +118,8 @@ func TestSecretInfoToRepoBindings(t *testing.T) { repos []*gh.Repository wantLen int wantFQNs []string - wantParentNil []bool // parallel to wantFQNs + wantNames []string // parallel to wantFQNs; nil slice skips the check + wantParentNil []bool // parallel to wantFQNs }{ { name: "valid repo produces binding with populated parent", @@ -116,8 +128,26 @@ func TestSecretInfoToRepoBindings(t *testing.T) { }, wantLen: 1, wantFQNs: []string{"github.com/truffle-sandbox/my-repo"}, + wantNames: []string{"my-repo"}, wantParentNil: []bool{false}, }, + { + name: "empty repo name falls back to full name (CSM-1554: Name must be non-empty)", + repos: []*gh.Repository{ + {Name: strPtr(""), FullName: strPtr("truffle-sandbox/ghost-repo"), Owner: owner}, + }, + wantLen: 1, + wantFQNs: []string{"github.com/truffle-sandbox/ghost-repo"}, + wantNames: []string{"truffle-sandbox/ghost-repo"}, + wantParentNil: []bool{false}, + }, + { + name: "both name and full name empty: repo is skipped", + repos: []*gh.Repository{ + {Name: strPtr(""), FullName: strPtr(""), Owner: owner}, + }, + wantLen: 0, + }, { name: "nil owner: repo is still included with nil parent", repos: []*gh.Repository{ @@ -169,6 +199,14 @@ func TestSecretInfoToRepoBindings(t *testing.T) { t.Errorf("binding[%d].Parent nil = %v, want %v", i, gotNil, wantNil) } } + for i, name := range tt.wantNames { + if i >= len(got) { + break + } + if got[i].Resource.Name != name { + t.Errorf("binding[%d].Name = %q, want %q", i, got[i].Resource.Name, name) + } + } }) } } @@ -394,7 +432,8 @@ func TestAnalyzer_Analyze(t *testing.T) { "Metadata": { "owner": "sirdetectsalot", "expiration": "2026-03-24T15:27:38+05:00", - "type": "Fine-Grained GitHub Personal Access Token" + "type": "Fine-Grained GitHub Personal Access Token", + "is_fine_grained": true } }`, }, @@ -442,7 +481,8 @@ func TestAnalyzer_Analyze(t *testing.T) { "Metadata": { "owner": "truffle-sandbox", "expiration": "0001-01-01T00:00:00Z", - "type": "Classic GitHub Personal Access Token" + "type": "Classic GitHub Personal Access Token", + "is_fine_grained": false } }`, wantErr: false, diff --git a/pkg/analyzer/analyzers/gitlab/gitlab.go b/pkg/analyzer/analyzers/gitlab/gitlab.go index 62ab2a587fa4..4fb26058376d 100644 --- a/pkg/analyzer/analyzers/gitlab/gitlab.go +++ b/pkg/analyzer/analyzers/gitlab/gitlab.go @@ -155,7 +155,7 @@ func getPersonalAccessToken(cfg *config.Config, key, host string) (AccessTokenJS return tokens, resp.StatusCode, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if err := json.NewDecoder(resp.Body).Decode(&tokens); err != nil { return tokens, resp.StatusCode, err } @@ -183,7 +183,7 @@ func getAccessibleProjects(cfg *config.Config, key, host string) ([]ProjectsJSON return projects, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() bodyBytes, err := io.ReadAll(resp.Body) if err != nil { @@ -197,7 +197,7 @@ func getAccessibleProjects(cfg *config.Config, key, host string) ([]ProjectsJSON if err := json.NewDecoder(newBody()).Decode(&projects); err != nil { var e ErrorJSON if err := json.NewDecoder(newBody()).Decode(&e); err == nil { - return projects, fmt.Errorf("Insufficient Scope to query for projects. We need api or read_api permissions.") + return projects, errors.New("insufficient scope to query for projects: we need api or read_api permissions") } return projects, err } @@ -219,7 +219,7 @@ func getMetadata(cfg *config.Config, key, host string) (MetadataJSON, error) { return metadata, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() bodyBytes, err := io.ReadAll(resp.Body) if err != nil { @@ -239,7 +239,7 @@ func getMetadata(cfg *config.Config, key, host string) (MetadataJSON, error) { if err := json.NewDecoder(newBody()).Decode(&e); err != nil { return metadata, err } - return metadata, fmt.Errorf("Insufficient Scope to query for metadata. We need read_user, ai_features, api or read_api permissions.") + return metadata, errors.New("insufficient scope to query for metadata: we need read_user, ai_features, api or read_api permissions") } return metadata, nil @@ -258,7 +258,7 @@ func AnalyzePermissions(cfg *config.Config, key string, host string) (*SecretInf return nil, err } if statusCode != http.StatusOK { - return nil, fmt.Errorf("Invalid GitLab Access Token") + return nil, errors.New("invalid GitLab access token") } meta, err := getMetadata(cfg, key, host) diff --git a/pkg/analyzer/analyzers/groq/groq.go b/pkg/analyzer/analyzers/groq/groq.go index 1e345cfd14a6..1e4f08d5d899 100644 --- a/pkg/analyzer/analyzers/groq/groq.go +++ b/pkg/analyzer/analyzers/groq/groq.go @@ -121,7 +121,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult { result := analyzers.AnalyzerResult{ AnalyzerType: analyzers.AnalyzerTypeGroq, Metadata: map[string]any{"Valid_Key": info.Valid}, - Bindings: make([]analyzers.Binding, len(info.GroqResources)), + Bindings: make([]analyzers.Binding, 0, len(info.GroqResources)), } // extract information to create bindings and append to result bindings diff --git a/pkg/analyzer/analyzers/groq/groq_test.go b/pkg/analyzer/analyzers/groq/groq_test.go index ef73d217b933..fef55a02bde7 100644 --- a/pkg/analyzer/analyzers/groq/groq_test.go +++ b/pkg/analyzer/analyzers/groq/groq_test.go @@ -12,6 +12,91 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) +func resource(id, name, resourceType string) GroqResource { + return GroqResource{ + ID: id, + Name: name, + Type: resourceType, + Permission: PermissionStrings[FullAccess], + Metadata: map[string]string{"status": "completed"}, + } +} + +func TestSecretInfoToAnalyzerResult(t *testing.T) { + tests := []struct { + name string + resources []GroqResource + wantLen int + wantNames []string + }{ + { + name: "one resource produces exactly one binding", + resources: []GroqResource{resource("batch_123", "batch_123", "batch")}, + wantLen: 1, + wantNames: []string{"batch_123"}, + }, + { + name: "three resources produce exactly three bindings, not six", + resources: []GroqResource{ + resource("batch_123", "batch_123", "batch"), + resource("file_456", "training.jsonl", "file"), + resource("file_789", "eval.jsonl", "file"), + }, + wantLen: 3, + wantNames: []string{"batch_123", "training.jsonl", "eval.jsonl"}, + }, + { + name: "no resources: no bindings", + resources: nil, + wantLen: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := &SecretInfo{ + Valid: true, + GroqResources: tt.resources, + } + + result := secretInfoToAnalyzerResult(info) + if result == nil { + t.Fatal("secretInfoToAnalyzerResult() returned nil") + } + + if len(result.Bindings) != tt.wantLen { + t.Errorf("got %d bindings, want %d", len(result.Bindings), tt.wantLen) + } + + // Mirrors the four proto min_len constraints so this test fails for + // the same reason the enterprise API does. + for i, binding := range result.Bindings { + if binding.Resource.Name == "" { + t.Errorf("binding[%d].Resource.Name is empty", i) + } + if binding.Resource.Type == "" { + t.Errorf("binding[%d].Resource.Type is empty", i) + } + if binding.Resource.FullyQualifiedName == "" { + t.Errorf("binding[%d].Resource.FullyQualifiedName is empty", i) + } + if binding.Permission.Value == "" { + t.Errorf("binding[%d].Permission.Value is empty", i) + } + } + + for i, name := range tt.wantNames { + if i >= len(result.Bindings) { + break + } + if result.Bindings[i].Resource.Name != name { + t.Errorf("binding[%d].Resource.Name = %q, want %q", i, result.Bindings[i].Resource.Name, name) + } + } + }) + } +} + func TestAnalyzer_Analyze(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() diff --git a/pkg/analyzer/analyzers/huggingface/huggingface.go b/pkg/analyzer/analyzers/huggingface/huggingface.go index b7ca38fc7a38..0473f09a5354 100644 --- a/pkg/analyzer/analyzers/huggingface/huggingface.go +++ b/pkg/analyzer/analyzers/huggingface/huggingface.go @@ -5,6 +5,7 @@ package huggingface import ( "encoding/json" "fmt" + "io" "net/http" "os" "strings" @@ -30,17 +31,17 @@ type Analyzer struct { func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeHuggingFace } -func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { +func (a Analyzer) Analyze(ctx context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok || key == "" { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", fmt.Errorf("key not found in credentialInfo"), - ) + err := fmt.Errorf("key not found in credentialInfo") + ctx.Logger().Error(err, "huggingface credentials missing key") + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", err) } - info, err := AnalyzePermissions(a.Cfg, key) + info, err := AnalyzePermissions(ctx, a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -90,32 +91,33 @@ func bakeUnfineGrainedBindings(allModels []Model, tokenJSON HFTokenJSON) []analy return bindings } -// finegrained scopes are grouped by org, user or model. -func bakefineGrainedBindings(allModels []Model, tokenJSON HFTokenJSON) []analyzers.Binding { - // this section will extract the relevant permissions for each entity and store them in a map +// bakefineGrainedBindings resolves each model's permission from the token's +// fine-grained scopes (grouped by org, user, or model). +func bakefineGrainedBindings(allModels []Model, tokenJSON HFTokenJSON) ([]analyzers.Binding, []analyzers.Resource) { var nameToPermissions = make(map[string]analyzers.Permission) for _, permission := range tokenJSON.Auth.AccessToken.FineGrained.Scoped { privs := analyzers.Permission{ Value: string(analyzers.NONE), } for _, perm := range permission.Permissions { - if perm == "repo.content.read" { + switch perm { + case "repo.content.read": privs.Value = string(analyzers.READ) - } else if perm == "repo.write" { + case "repo.write": privs.Value = string(analyzers.WRITE) } } - if permission.Entity.Type == "user" || permission.Entity.Type == "org" { + switch permission.Entity.Type { + case "user", "org": nameToPermissions[permission.Entity.Name] = privs - } else if permission.Entity.Type == "model" { + case "model": nameToPermissions[modelNameLookup(allModels, permission.Entity.ID)] = privs } } - bindings := make([]analyzers.Binding, len(allModels)) - for idx, model := range allModels { - - // Add Read Privs to All Models + var bindings []analyzers.Binding + var unbounded []analyzers.Resource + for _, model := range allModels { modelResource := analyzers.Resource{ Name: model.Name, FullyQualifiedName: "huggingface.com/model/" + model.ID, @@ -126,22 +128,27 @@ func bakefineGrainedBindings(allModels []Model, tokenJSON HFTokenJSON) []analyze } var perm analyzers.Permission - // get username/orgname for each model and apply those permissions modelUsername := strings.Split(model.Name, "/")[0] if permissions, ok := nameToPermissions[modelUsername]; ok { perm = permissions } - // override model permissions with repo-specific permissions if permissions, ok := nameToPermissions[model.Name]; ok { perm = permissions } - bindings[idx] = analyzers.Binding{ + // Models without an explicit fine-grained scope are visible to the + // token but have no bound permission; they belong in UnboundedResources. + if perm.Value == "" { + unbounded = append(unbounded, modelResource) + continue + } + + bindings = append(bindings, analyzers.Binding{ Resource: modelResource, Permission: perm, - } + }) } - return bindings + return bindings, unbounded } func bakeOrganizationBindings(tokenJSON HFTokenJSON) []analyzers.Binding { @@ -256,7 +263,9 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult { result.Bindings = make([]analyzers.Binding, 0) if info.Token.Auth.AccessToken.Type == FINEGRAINED { - result.Bindings = append(result.Bindings, bakefineGrainedBindings(info.Models, info.Token)...) + fgBindings, fgUnbounded := bakefineGrainedBindings(info.Models, info.Token) + result.Bindings = append(result.Bindings, fgBindings...) + result.UnboundedResources = append(result.UnboundedResources, fgUnbounded...) result.Bindings = append(result.Bindings, bakeOrganizationBindings(info.Token)...) result.Bindings = append(result.Bindings, bakeUserBindings(info.Token)...) } else { @@ -309,13 +318,14 @@ type Model struct { // getModelsByAuthor calls the HF API /models endpoint with the author query param // returns a list of models and an error -func getModelsByAuthor(cfg *config.Config, key string, author string) ([]Model, error) { +func getModelsByAuthor(ctx context.Context, cfg *config.Config, key string, author string) ([]Model, error) { var modelsJSON []Model // create a new request client := analyzers.NewAnalyzeClient(cfg) - req, err := http.NewRequest("GET", "https://huggingface.co/api/models", nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/models", nil) if err != nil { + ctx.Logger().Error(err, "failed to create huggingface models request", "author", author) return modelsJSON, err } @@ -330,14 +340,25 @@ func getModelsByAuthor(cfg *config.Config, key string, author string) ([]Model, // send the request resp, err := client.Do(req) if err != nil { + ctx.Logger().Error(err, "failed to send huggingface models request", "author", author) return modelsJSON, err } // defer the response body closing - defer resp.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("unexpected status code: %d while fetching models for author %s", resp.StatusCode, author) + ctx.Logger().Error(err, "unexpected status code while fetching huggingface models", "author", author, "status_code", resp.StatusCode) + return modelsJSON, err + } // read response if err := json.NewDecoder(resp.Body).Decode(&modelsJSON); err != nil { + ctx.Logger().Error(err, "failed to decode huggingface models response", "author", author) return modelsJSON, err } return modelsJSON, nil @@ -345,13 +366,14 @@ func getModelsByAuthor(cfg *config.Config, key string, author string) ([]Model, // getTokenInfo calls the HF API /whoami-v2 endpoint to get the token info // returns the token info, a boolean indicating token validity, and an error -func getTokenInfo(cfg *config.Config, key string) (HFTokenJSON, bool, error) { +func getTokenInfo(ctx context.Context, cfg *config.Config, key string) (HFTokenJSON, bool, error) { var tokenJSON HFTokenJSON // create a new request client := analyzers.NewAnalyzeClient(cfg) - req, err := http.NewRequest("GET", "https://huggingface.co/api/whoami-v2", nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil) if err != nil { + ctx.Logger().Error(err, "failed to create huggingface whoami request") return tokenJSON, false, err } @@ -361,19 +383,25 @@ func getTokenInfo(cfg *config.Config, key string) (HFTokenJSON, bool, error) { // send the request resp, err := client.Do(req) if err != nil { + ctx.Logger().Error(err, "failed to send huggingface whoami request") return tokenJSON, false, err } + // defer the response body closing + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + // check if the response is 200 - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { + ctx.Logger().V(2).Info("huggingface token invalid while fetching whoami", "status_code", resp.StatusCode) return tokenJSON, false, nil } - // defer the response body closing - defer resp.Body.Close() - // read response if err := json.NewDecoder(resp.Body).Decode(&tokenJSON); err != nil { + ctx.Logger().Error(err, "failed to decode huggingface whoami response") return tokenJSON, true, err } return tokenJSON, true, nil @@ -384,20 +412,22 @@ type SecretInfo struct { Models []Model } -func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { +func AnalyzePermissions(ctx context.Context, cfg *config.Config, key string) (*SecretInfo, error) { // get token info - token, success, err := getTokenInfo(cfg, key) + token, success, err := getTokenInfo(ctx, cfg, key) if err != nil { return nil, err } if !success { - return nil, fmt.Errorf("Invalid HuggingFace Access Token") + err := fmt.Errorf("invalid HuggingFace Access Token") + ctx.Logger().V(2).Info("huggingface access token validation failed") + return nil, err } // get all models by username var allModels []Model - userModels, err := getModelsByAuthor(cfg, key, token.Username) + userModels, err := getModelsByAuthor(ctx, cfg, key, token.Username) if err != nil { return nil, err } @@ -405,7 +435,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { // get all models from all orgs for _, org := range token.Orgs { - orgModels, err := getModelsByAuthor(cfg, key, org.Name) + orgModels, err := getModelsByAuthor(ctx, cfg, key, org.Name) if err != nil { return nil, err } @@ -420,7 +450,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { // AnalyzeAndPrintPermissions prints the permissions of a HuggingFace API key func AnalyzeAndPrintPermissions(cfg *config.Config, key string) { - info, err := AnalyzePermissions(cfg, key) + info, err := AnalyzePermissions(context.Background(), cfg, key) if err != nil { color.Red("[x] Error: %s", err.Error()) return @@ -609,15 +639,17 @@ func printAccessibleModels(allModels []Model, tokenJSON HFTokenJSON) { read := false write := false for _, perm := range permission.Permissions { - if perm == "repo.content.read" { + switch perm { + case "repo.content.read": read = true - } else if perm == "repo.write" { + case "repo.write": write = true } } - if permission.Entity.Type == "user" || permission.Entity.Type == "org" { + switch permission.Entity.Type { + case "user", "org": nameToPermissions[permission.Entity.Name] = Permissions{Read: read, Write: write} - } else if permission.Entity.Type == "model" { + case "model": nameToPermissions[modelNameLookup(allModels, permission.Entity.ID)] = Permissions{Read: read, Write: write} } } diff --git a/pkg/analyzer/analyzers/huggingface/huggingface_test.go b/pkg/analyzer/analyzers/huggingface/huggingface_test.go index eba22f70efd2..0502807b845f 100644 --- a/pkg/analyzer/analyzers/huggingface/huggingface_test.go +++ b/pkg/analyzer/analyzers/huggingface/huggingface_test.go @@ -118,3 +118,132 @@ func TestAnalyzer_Analyze(t *testing.T) { }) } } + +// Test_secretInfoToAnalyzerResult_FineGrainedUnboundedResources verifies that +// models outside the token's fine-grained permission scopes land in +// UnboundedResources rather than producing bindings with empty Permission.Value. +func Test_secretInfoToAnalyzerResult_FineGrainedUnboundedResources(t *testing.T) { + info := &SecretInfo{ + Token: HFTokenJSON{ + Username: "testuser", + Name: "Test User", + Auth: struct { + AccessToken struct { + Name string `json:"displayName"` + Type string `json:"role"` + CreatedAt string `json:"createdAt"` + FineGrained struct { + Global []string `json:"global"` + Scoped []struct { + Entity struct { + Type string `json:"type"` + Name string `json:"name"` + ID string `json:"_id"` + } `json:"entity"` + Permissions []string `json:"permissions"` + } `json:"scoped"` + } `json:"fineGrained"` + } + }{ + AccessToken: struct { + Name string `json:"displayName"` + Type string `json:"role"` + CreatedAt string `json:"createdAt"` + FineGrained struct { + Global []string `json:"global"` + Scoped []struct { + Entity struct { + Type string `json:"type"` + Name string `json:"name"` + ID string `json:"_id"` + } `json:"entity"` + Permissions []string `json:"permissions"` + } `json:"scoped"` + } `json:"fineGrained"` + }{ + Name: "test-token", + Type: FINEGRAINED, + FineGrained: struct { + Global []string `json:"global"` + Scoped []struct { + Entity struct { + Type string `json:"type"` + Name string `json:"name"` + ID string `json:"_id"` + } `json:"entity"` + Permissions []string `json:"permissions"` + } `json:"scoped"` + }{ + Scoped: []struct { + Entity struct { + Type string `json:"type"` + Name string `json:"name"` + ID string `json:"_id"` + } `json:"entity"` + Permissions []string `json:"permissions"` + }{ + { + Entity: struct { + Type string `json:"type"` + Name string `json:"name"` + ID string `json:"_id"` + }{Type: "user", Name: "testuser"}, + Permissions: []string{"repo.content.read"}, + }, + }, + }, + }, + }, + }, + Models: []Model{ + {Name: "testuser/scoped-model", ID: "abc123", Private: true}, + {Name: "other-org/unscoped-model", ID: "def456", Private: false}, + {Name: "another-org/also-unscoped", ID: "ghi789", Private: false}, + }, + } + + result := secretInfoToAnalyzerResult(info) + + // The scoped model should appear as a binding with a non-empty permission. + if len(result.Bindings) == 0 { + t.Fatal("expected at least one binding for the scoped model") + } + foundScoped := false + for _, b := range result.Bindings { + if b.Resource.Name == "testuser/scoped-model" { + foundScoped = true + if b.Permission.Value == "" { + t.Error("scoped model binding has empty Permission.Value") + } + } + if b.Permission.Value == "" { + t.Errorf("binding for %q has empty Permission.Value (would fail proto validation)", b.Resource.Name) + } + } + if !foundScoped { + t.Error("expected testuser/scoped-model in Bindings") + } + + // Unscoped models should appear in UnboundedResources, not Bindings. + unscopedNames := map[string]bool{ + "other-org/unscoped-model": false, + "another-org/also-unscoped": false, + } + for _, r := range result.UnboundedResources { + if _, want := unscopedNames[r.Name]; want { + unscopedNames[r.Name] = true + } + } + for name, found := range unscopedNames { + if !found { + t.Errorf("expected %q in UnboundedResources, but it was missing", name) + } + } + + // Verify unscoped models are NOT in Bindings. + for _, b := range result.Bindings { + if _, isUnscoped := unscopedNames[b.Resource.Name]; isUnscoped { + t.Errorf("unscoped model %q should not appear in Bindings", b.Resource.Name) + } + } +} diff --git a/pkg/analyzer/analyzers/jira/requests.go b/pkg/analyzer/analyzers/jira/requests.go index bac24b9133e7..43bb6edc8e20 100644 --- a/pkg/analyzer/analyzers/jira/requests.go +++ b/pkg/analyzer/analyzers/jira/requests.go @@ -428,13 +428,12 @@ func captureAuditLogs(client *http.Client, domain, email, token string, secretIn } func handleStatusCode(statusCode int, endpoint string) error { - switch { - case statusCode == http.StatusOK: + switch statusCode { + case http.StatusOK: return nil - case statusCode == http.StatusBadRequest: + case http.StatusBadRequest: return fmt.Errorf("bad request for API: %s", endpoint) - case statusCode == http.StatusUnauthorized, statusCode == http.StatusForbidden, - statusCode == http.StatusNotFound, statusCode == http.StatusConflict: + case http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusConflict: return nil default: return fmt.Errorf("unexpected status code: %d for API: %s", statusCode, endpoint) diff --git a/pkg/analyzer/analyzers/mailchimp/mailchimp.go b/pkg/analyzer/analyzers/mailchimp/mailchimp.go index 514f25d3d565..316c232d8149 100644 --- a/pkg/analyzer/analyzers/mailchimp/mailchimp.go +++ b/pkg/analyzer/analyzers/mailchimp/mailchimp.go @@ -146,7 +146,7 @@ func getMetadata(cfg *config.Config, key string) (MetadataJSON, error) { return metadata, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { return metadata, err @@ -177,7 +177,7 @@ func getDomains(cfg *config.Config, key string) (DomainsJSON, error) { return domains, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if err := json.NewDecoder(resp.Body).Decode(&domains); err != nil { return domains, err @@ -198,7 +198,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { return nil, err } if metadata.AccountID == "" { - return nil, fmt.Errorf("Invalid Mailchimp API key") + return nil, fmt.Errorf("invalid Mailchimp API key") } // get sending domains diff --git a/pkg/analyzer/analyzers/mailgun/requests.go b/pkg/analyzer/analyzers/mailgun/requests.go index ab7da22da167..8f9f653333fd 100644 --- a/pkg/analyzer/analyzers/mailgun/requests.go +++ b/pkg/analyzer/analyzers/mailgun/requests.go @@ -53,7 +53,7 @@ func getDomains(client *http.Client, apiKey string, secretInfo *SecretInfo) erro if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { return fmt.Errorf("invalid Mailgun API key") @@ -83,7 +83,7 @@ func getKeys(client *http.Client, apiKey string, secretInfo *SecretInfo) error { if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { return fmt.Errorf("invalid Mailgun API key") diff --git a/pkg/analyzer/analyzers/mux/models.go b/pkg/analyzer/analyzers/mux/models.go index e7d3b52afa93..87e978dd1ec3 100644 --- a/pkg/analyzer/analyzers/mux/models.go +++ b/pkg/analyzer/analyzers/mux/models.go @@ -56,9 +56,10 @@ func (info *secretInfo) addPermission(resourceType ResourceType, permission stri return } - if permission == "read" { + switch permission { + case "read": info.Permissions[resourceType] = Read - } else if permission == "write" { + case "write": info.Permissions[resourceType] = FullAccess } } diff --git a/pkg/analyzer/analyzers/mysql/mysql.go b/pkg/analyzer/analyzers/mysql/mysql.go index 213ed6757cf1..131634911f87 100644 --- a/pkg/analyzer/analyzers/mysql/mysql.go +++ b/pkg/analyzer/analyzers/mysql/mysql.go @@ -292,7 +292,7 @@ func AnalyzePermissions(cfg *config.Config, connectionStr string) (*SecretInfo, if err != nil { return nil, fmt.Errorf("connecting to the MySQL database: %w", err) } - defer db.Close() + defer func() { _ = db.Close() }() // Get the current user user, err := getUser(db) @@ -434,7 +434,7 @@ func getDatabases(db *sql.DB, databases map[string]*Database) error { if err != nil { return err } - defer rows.Close() + defer func() { _ = rows.Close() }() for rows.Next() { var dbName string @@ -462,7 +462,7 @@ func getTables(db *sql.DB, databases map[string]*Database) error { if err != nil { return err } - defer rows.Close() + defer func() { _ = rows.Close() }() for rows.Next() { var dbName string @@ -486,7 +486,7 @@ func getRoutines(db *sql.DB, databases map[string]*Database) error { if err != nil { return err } - defer rows.Close() + defer func() { _ = rows.Close() }() for rows.Next() { var dbName string @@ -513,7 +513,7 @@ func getGrants(db *sql.DB) ([]string, error) { if err != nil { return nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() var grants []string for rows.Next() { @@ -568,7 +568,7 @@ func processGrant(grant string, databases map[string]*Database, globalPrivs *Glo // Split on " ON " parts := strings.Split(grant, " ON ") if len(parts) < 2 { - return fmt.Errorf("Error processing grant: %s", grant) + return fmt.Errorf("error processing grant: %s", grant) } // Put privs in a slice diff --git a/pkg/analyzer/analyzers/notion/notion.go b/pkg/analyzer/analyzers/notion/notion.go index 35c7e0c1d718..fe78cc20c733 100644 --- a/pkg/analyzer/analyzers/notion/notion.go +++ b/pkg/analyzer/analyzers/notion/notion.go @@ -30,13 +30,11 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeNot func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -143,7 +141,7 @@ func (h *HttpStatusTest) RunTest(cfg *config.Config, headers map[string]string) if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { @@ -330,7 +328,7 @@ func getBotInfo(client *http.Client, key string) (*bot, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: @@ -368,7 +366,7 @@ func getWorkspaceUsers(client *http.Client, key string) ([]user, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/analyzer/analyzers/openai/openai.go b/pkg/analyzer/analyzers/openai/openai.go index d7057db0546c..69c4410a585f 100644 --- a/pkg/analyzer/analyzers/openai/openai.go +++ b/pkg/analyzer/analyzers/openai/openai.go @@ -31,8 +31,7 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeOpe func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { info, err := AnalyzePermissions(a.Cfg, credInfo["key"]) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -214,7 +213,7 @@ func openAIRequest(cfg *config.Config, method string, url string, key string, da return nil, nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() outBody, err := io.ReadAll(resp.Body) if err != nil { diff --git a/pkg/analyzer/analyzers/opsgenie/opsgenie.go b/pkg/analyzer/analyzers/opsgenie/opsgenie.go index 6c13a92ef4a6..90c3bea580a8 100644 --- a/pkg/analyzer/analyzers/opsgenie/opsgenie.go +++ b/pkg/analyzer/analyzers/opsgenie/opsgenie.go @@ -30,13 +30,11 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeOps func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -159,7 +157,7 @@ func (h *HttpStatusTest) RunTest(cfg *config.Config, headers map[string]string) if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { @@ -231,7 +229,7 @@ func getUserList(cfg *config.Config, key string) ([]User, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Decode response body var userList UsersJSON diff --git a/pkg/analyzer/analyzers/plaid/plaid.go b/pkg/analyzer/analyzers/plaid/plaid.go index fc44e98100c5..5b2f936c2992 100644 --- a/pkg/analyzer/analyzers/plaid/plaid.go +++ b/pkg/analyzer/analyzers/plaid/plaid.go @@ -108,7 +108,7 @@ func getPlaidAccounts(client *http.Client, clientID string, secret string, acces if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("received non-OK HTTP status: %d", resp.StatusCode) diff --git a/pkg/analyzer/analyzers/planetscale/planetscale.go b/pkg/analyzer/analyzers/planetscale/planetscale.go index 9d356be9c63a..ddbbd395302a 100644 --- a/pkg/analyzer/analyzers/planetscale/planetscale.go +++ b/pkg/analyzer/analyzers/planetscale/planetscale.go @@ -137,7 +137,7 @@ func (h *HttpStatusTest) RunTest(cfg *config.Config, headers map[string]string, if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { @@ -539,7 +539,7 @@ func sendGetRequest(cfg *config.Config, id, key, url string, responseObj interfa if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch resp.StatusCode { diff --git a/pkg/analyzer/analyzers/postgres/postgres.go b/pkg/analyzer/analyzers/postgres/postgres.go index 3d5a8265ae46..ae7ce568dd82 100644 --- a/pkg/analyzer/analyzers/postgres/postgres.go +++ b/pkg/analyzer/analyzers/postgres/postgres.go @@ -6,8 +6,8 @@ import ( "database/sql" "errors" "fmt" + regexp "github.com/wasilibs/go-re2" "os" - "regexp" "strings" "github.com/fatih/color" @@ -27,13 +27,15 @@ type Analyzer struct { func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypePostgres } -func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { +func (a Analyzer) Analyze(ctx context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { uri, ok := credInfo["connection_string"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("connection string not found in credInfo")) + err := errors.New("connection string not found in credInfo") + ctx.Logger().Error(err, "postgres credentials missing connection string") + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", err) } - info, err := AnalyzePermissions(a.Cfg, uri) + info, err := AnalyzePermissions(ctx, a.Cfg, uri) if err != nil { return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceDatabase, "", err) } @@ -239,7 +241,7 @@ func AnalyzeAndPrintPermissions(cfg *config.Config, connectionStr string) { return } - info, err := AnalyzePermissions(cfg, connectionStr) + info, err := AnalyzePermissions(context.Background(), cfg, connectionStr) if err != nil { color.Red("[x] Error: %s", err.Error()) return @@ -263,32 +265,34 @@ func AnalyzeAndPrintPermissions(cfg *config.Config, connectionStr string) { } } -func AnalyzePermissions(cfg *config.Config, connectionStr string) (*SecretInfo, error) { +func AnalyzePermissions(ctx context.Context, cfg *config.Config, connectionStr string) (*SecretInfo, error) { connStr, err := pq.ParseURL(string(connectionStr)) if err != nil { - return nil, fmt.Errorf("failed to parse Postgres connection string: %w", err) + err = fmt.Errorf("failed to parse Postgres connection string: %w", err) + ctx.Logger().Error(err, "invalid postgres connection string") + return nil, err } parts := connStrPartPattern.FindAllStringSubmatch(connStr, -1) params := make(map[string]string, len(parts)) for _, part := range parts { params[part[1]] = part[2] } - db, err := createConnection(params, "") + db, err := createConnection(ctx, params, "") if err != nil { return nil, fmt.Errorf("failed to connect to Postgres database: %w", err) } - defer db.Close() + defer func() { _ = db.Close() }() - role, privs, err := getUserPrivs(db) + role, privs, err := getUserPrivs(ctx, db) if err != nil { return nil, fmt.Errorf("failed to retrieve user privileges: %w", err) } - currentUser, dbs, err := getDBPrivs(db) + currentUser, dbs, err := getDBPrivs(ctx, db) if err != nil { return nil, fmt.Errorf("failed to retrieve database privileges: %w", err) } - tablePrivs, err := getTablePrivs(params, buildSliceDBNames(dbs)) + tablePrivs, err := getTablePrivs(ctx, params, buildSliceDBNames(dbs)) if err != nil { return nil, fmt.Errorf("failed to retrieve table privileges: %w", err) } @@ -313,7 +317,7 @@ func isErrorDatabaseNotFound(err error, dbName string, user string) bool { return false } -func createConnection(params map[string]string, database string) (*sql.DB, error) { +func createConnection(ctx context.Context, params map[string]string, database string) (*sql.DB, error) { if sslmode := params[pg_sslmode]; sslmode == pg_sslmode_allow || sslmode == pg_sslmode_prefer { // pq doesn't support 'allow' or 'prefer'. If we find either of them, we'll just ignore it. This will trigger // the same logic that is run if no sslmode is set at all (which mimics 'prefer', which is the default). @@ -331,31 +335,35 @@ func createConnection(params map[string]string, database string) (*sql.DB, error db, err := sql.Open("postgres", connStr) if err != nil { + ctx.Logger().Error(err, "failed to open postgres connection", "database", database) return nil, err } - err = db.Ping() + err = db.PingContext(ctx) switch { case err == nil: return db, nil case strings.Contains(err.Error(), "password authentication failed"): - return nil, errors.New("password authentication failed") + err := errors.New("password authentication failed") + ctx.Logger().V(2).Info("postgres password authentication failed", "database", database) + return nil, err case errors.Is(err, pq.ErrSSLNotSupported) && params[pg_sslmode] == "": // If the sslmode is unset, then either it was unset in the candidate secret, or we've intentionally unset it // because it was specified as 'allow' or 'prefer', neither of which pq supports. In all of these cases, non-SSL // connections are acceptable, so now we try a connection without SSL. params[pg_sslmode] = pg_sslmode_disable defer delete(params, pg_sslmode) // We want to return with the original params map intact (for ExtraData) - return createConnection(params, database) + return createConnection(ctx, params, database) case isErrorDatabaseNotFound(err, params[pg_dbname], params[pg_user]): color.Green("[!] Successfully connected to Postgres database.") return nil, err default: + ctx.Logger().Error(err, "failed to ping postgres database", "database", database) return nil, err } } -func getUserPrivs(db *sql.DB) (string, map[string]bool, error) { +func getUserPrivs(ctx context.Context, db *sql.DB) (string, map[string]bool, error) { // Prepare the SQL statement query := `SELECT rolname AS role_name, rolsuper AS is_superuser, @@ -368,28 +376,31 @@ func getUserPrivs(db *sql.DB) (string, map[string]bool, error) { FROM pg_roles WHERE rolname = current_user;` // Execute the SQL query - rows, err := db.Query(query) + rows, err := db.QueryContext(ctx, query) if err != nil { + ctx.Logger().Error(err, "failed to query postgres user privileges") return "", nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() var roleName string var isSuperuser, canInherit, canCreateRole, canCreateDB, canLogin, isReplicationRole, bypassesRLS bool // Iterate over the rows for rows.Next() { if err := rows.Scan(&roleName, &isSuperuser, &canInherit, &canCreateRole, &canCreateDB, &canLogin, &isReplicationRole, &bypassesRLS); err != nil { + ctx.Logger().Error(err, "failed to scan postgres user privileges row") return "", nil, err } } // Check for errors during iteration if err := rows.Err(); err != nil { + ctx.Logger().Error(err, "error iterating postgres user privileges rows") return "", nil, err } // Map roles to privileges - var mapRoles map[string]bool = map[string]bool{ + var mapRoles = map[string]bool{ "Superuser": isSuperuser, "Inheritance of Privs": canInherit, "Create Role": canCreateRole, @@ -402,7 +413,7 @@ func getUserPrivs(db *sql.DB) (string, map[string]bool, error) { return roleName, mapRoles, nil } -func getDBPrivs(db *sql.DB) (string, []DB, error) { +func getDBPrivs(ctx context.Context, db *sql.DB) (string, []DB, error) { query := ` SELECT d.datname AS database_name, @@ -423,11 +434,12 @@ func getDBPrivs(db *sql.DB) (string, []DB, error) { // Originally had WHERE NOT d.datistemplate AND d.datallowconn // Execute the query - rows, err := db.Query(query) + rows, err := db.QueryContext(ctx, query) if err != nil { + ctx.Logger().Error(err, "failed to query postgres database privileges") return "", nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() dbs := make([]DB, 0) @@ -438,6 +450,7 @@ func getDBPrivs(db *sql.DB) (string, []DB, error) { var canConnect, canCreate, canCreateTemp bool err := rows.Scan(&dbName, &owner, ¤tUser, &canConnect, &canCreate, &canCreateTemp) if err != nil { + ctx.Logger().Error(err, "failed to scan postgres database privileges row") return "", nil, err } @@ -453,6 +466,7 @@ func getDBPrivs(db *sql.DB) (string, []DB, error) { dbs = append(dbs, db) } if err = rows.Err(); err != nil { + ctx.Logger().Error(err, "error iterating postgres database privileges rows") return "", nil, err } @@ -501,25 +515,25 @@ func getDBWriter(db DB, current_user string) func(a ...interface{}) string { func buildSliceDBNames(dbs []DB) []string { var dbNames []string for _, db := range dbs { - if db.DBPrivs.Connect { + if db.Connect { dbNames = append(dbNames, db.DatabaseName) } } return dbNames } -func getTablePrivs(params map[string]string, databases []string) (map[string]map[string]*TableData, error) { +func getTablePrivs(ctx context.Context, params map[string]string, databases []string) (map[string]map[string]*TableData, error) { tablePrivileges := make(map[string]map[string]*TableData, 0) for _, dbase := range databases { // Connect to db - db, err := createConnection(params, dbase) + db, err := createConnection(ctx, params, dbase) if err != nil { - // color.Red("[x] Failed to connect to Postgres database: %s", dbase) + ctx.Logger().Error(err, "failed to connect to postgres database while fetching table privileges", "database", dbase) continue } - defer db.Close() + defer func() { _ = db.Close() }() // Get table privs query := ` @@ -539,17 +553,19 @@ func getTablePrivs(params map[string]string, databases []string) (map[string]map ` // Execute the query - rows, err := db.Query(query) + rows, err := db.QueryContext(ctx, query) if err != nil { + ctx.Logger().Error(err, "failed to query postgres table privileges", "database", dbase) return nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() // Iterate through the result set for rows.Next() { var database, table, priv, size, row_count string err := rows.Scan(&database, &table, &priv, &size, &row_count) if err != nil { + ctx.Logger().Error(err, "failed to scan postgres table privileges row", "database", dbase) return nil, err } @@ -587,9 +603,10 @@ func getTablePrivs(params map[string]string, databases []string) (map[string]map } } if err = rows.Err(); err != nil { + ctx.Logger().Error(err, "error iterating postgres table privileges rows", "database", dbase) return nil, err } - db.Close() + _ = db.Close() } return tablePrivileges, nil diff --git a/pkg/analyzer/analyzers/posthog/posthog.go b/pkg/analyzer/analyzers/posthog/posthog.go index 2706e988c863..81399e47ffff 100644 --- a/pkg/analyzer/analyzers/posthog/posthog.go +++ b/pkg/analyzer/analyzers/posthog/posthog.go @@ -37,13 +37,11 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypePos func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", errors.New("missing key in credInfo")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -166,7 +164,7 @@ func (h *HttpStatusTest) RunTest(cfg *config.Config, client *http.Client, domain if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { @@ -302,7 +300,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { // we need to determine if the key is for US or EU domain domain, user, err := resolveDomainAndUser(cfg, client, key) if err != nil { - return nil, fmt.Errorf("Invalid API Key: %w", err) + return nil, fmt.Errorf("invalid API key: %w", err) } info.user = user @@ -439,7 +437,7 @@ func resolveDomainAndUser(cfg *config.Config, client *http.Client, key string) ( if err != nil { return "", nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: @@ -476,7 +474,7 @@ func getOrganization(cfg *config.Config, client *http.Client, domain string, key if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/analyzer/analyzers/postman/postman.go b/pkg/analyzer/analyzers/postman/postman.go index 93a09ffca9d4..b3291be888a7 100644 --- a/pkg/analyzer/analyzers/postman/postman.go +++ b/pkg/analyzer/analyzers/postman/postman.go @@ -26,13 +26,11 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypePos func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", fmt.Errorf("missing key in credInfo"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", fmt.Errorf("missing key in credInfo")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -148,7 +146,7 @@ func getUserInfo(cfg *config.Config, key string) (UserInfoJSON, error) { } // read response - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // if status code is 200, decode response if resp.StatusCode == 200 { @@ -175,7 +173,7 @@ func getWorkspaces(cfg *config.Config, key string) (WorkspaceJSON, error) { } // read response - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // if status code is 200, decode response if resp.StatusCode == 200 { @@ -224,7 +222,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) { } if me.User.Username == "" { - return nil, fmt.Errorf("Invalid Postman API Key") + return nil, fmt.Errorf("invalid Postman API Key") } // get workspaces, if there is error user with empty workspaces will be returned diff --git a/pkg/analyzer/analyzers/privatekey/privatekey.go b/pkg/analyzer/analyzers/privatekey/privatekey.go index 02a63a0380da..51f82b7be055 100644 --- a/pkg/analyzer/analyzers/privatekey/privatekey.go +++ b/pkg/analyzer/analyzers/privatekey/privatekey.go @@ -5,8 +5,8 @@ package privatekey import ( "errors" "fmt" + regexp "github.com/wasilibs/go-re2" "os" - "regexp" "strings" "sync" "time" diff --git a/pkg/analyzer/analyzers/shopify/shopify.go b/pkg/analyzer/analyzers/shopify/shopify.go index ad1cafa92887..dd26bf38a686 100644 --- a/pkg/analyzer/analyzers/shopify/shopify.go +++ b/pkg/analyzer/analyzers/shopify/shopify.go @@ -213,7 +213,7 @@ func getShopInfo(cfg *config.Config, key string, store string) (ShopInfoJSON, er return shopInfo, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() err = json.NewDecoder(resp.Body).Decode(&shopInfo) if err != nil { @@ -252,7 +252,7 @@ func getAccessScopes(cfg *config.Config, key string, store string) (AccessScopes return accessScopes, resp.StatusCode, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() err = json.NewDecoder(resp.Body).Decode(&accessScopes) if err != nil { diff --git a/pkg/analyzer/analyzers/slack/slack.go b/pkg/analyzer/analyzers/slack/slack.go index a8d55f6e2b19..62a6b2893b32 100644 --- a/pkg/analyzer/analyzers/slack/slack.go +++ b/pkg/analyzer/analyzers/slack/slack.go @@ -141,7 +141,7 @@ func getSlackOAuthScopes(cfg *config.Config, key string) (scopes string, userDat if err != nil { return scopes, userData, err } - defer resp.Body.Close() // Close the response body when the function returns + defer func() { _ = resp.Body.Close() }() // Close the response body when the function returns // print body body, err := io.ReadAll(resp.Body) diff --git a/pkg/analyzer/analyzers/sourcegraph/sourcegraph.go b/pkg/analyzer/analyzers/sourcegraph/sourcegraph.go index 609d3aee0b77..9c8033797832 100644 --- a/pkg/analyzer/analyzers/sourcegraph/sourcegraph.go +++ b/pkg/analyzer/analyzers/sourcegraph/sourcegraph.go @@ -26,13 +26,11 @@ func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeSou func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) { key, ok := credInfo["key"] if !ok { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", fmt.Errorf("missing key in credInfo"), - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", fmt.Errorf("missing key in credInfo")) } info, err := AnalyzePermissions(a.Cfg, key) if err != nil { - return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err, - ) + return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err) } return secretInfoToAnalyzerResult(info), nil } @@ -116,7 +114,7 @@ func getUserInfo(cfg *config.Config, key string) (UserInfoJSON, error) { return userInfo, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() err = json.NewDecoder(resp.Body).Decode(&userInfo) if err != nil { @@ -151,7 +149,7 @@ func checkSiteAdmin(cfg *config.Config, key string) (bool, error) { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() var response GraphQLResponse diff --git a/pkg/analyzer/analyzers/square/square.go b/pkg/analyzer/analyzers/square/square.go index 104b2119020a..bb0a26a6dd32 100644 --- a/pkg/analyzer/analyzers/square/square.go +++ b/pkg/analyzer/analyzers/square/square.go @@ -173,7 +173,7 @@ func getPermissions(cfg *config.Config, key string) (PermissionsJSON, error) { return permissions, nil } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() err = json.NewDecoder(resp.Body).Decode(&permissions) if err != nil { @@ -210,7 +210,7 @@ func getUsers(cfg *config.Config, key string) (TeamJSON, error) { return team, nil } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() err = json.NewDecoder(resp.Body).Decode(&team) if err != nil { diff --git a/pkg/analyzer/analyzers/stripe/stripe.go b/pkg/analyzer/analyzers/stripe/stripe.go index 65325406ecab..d100182e589e 100644 --- a/pkg/analyzer/analyzers/stripe/stripe.go +++ b/pkg/analyzer/analyzers/stripe/stripe.go @@ -162,7 +162,7 @@ func (h *HttpStatusTest) RunTest(cfg *config.Config, headers map[string]string) if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status code switch { @@ -195,7 +195,7 @@ func checkKeyType(key string) (string, error) { } else if strings.HasPrefix(key, RESTRICTED_PREFIX) { return RESTRICTED, nil } - return "", errors.New("Invalid Stripe key format") + return "", errors.New("invalid Stripe key format") } func checkKeyEnv(key string) (string, error) { @@ -226,7 +226,7 @@ func checkValidity(cfg *config.Config, key string) (bool, error) { if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check the response. Valid is 200 (secret/restricted) or 403 (restricted) if resp.StatusCode == 200 || resp.StatusCode == 403 { @@ -290,15 +290,17 @@ func AnalyzeAndPrintPermissions(cfg *config.Config, key string) { color.Green("[!] Valid Stripe API Key\n\n") - if info.KeyType == SECRET { + switch info.KeyType { + case SECRET: color.Green("[i] Key Type: %s", info.KeyType) - } else if info.KeyType == RESTRICTED { + case RESTRICTED: color.Yellow("[i] Key Type: %s", info.KeyType) } - if info.KeyEnv == LIVE { + switch info.KeyEnv { + case LIVE: color.Green("[i] Key Environment: %s", info.KeyEnv) - } else if info.KeyEnv == TEST { + case TEST: color.Red("[i] Key Environment: %s", info.KeyEnv) } diff --git a/pkg/analyzer/analyzers/twilio/twilio.go b/pkg/analyzer/analyzers/twilio/twilio.go index 87229f662bd8..3a92d2bc0ed2 100644 --- a/pkg/analyzer/analyzers/twilio/twilio.go +++ b/pkg/analyzer/analyzers/twilio/twilio.go @@ -49,7 +49,8 @@ func (a *Analyzer) Analyze(ctx context.Context, credentialInfo map[string]string } var permissions []Permission - if info.AccountStatusCode == 200 { + switch info.AccountStatusCode { + case 200: permissions = []Permission{ AccountManagementRead, AccountManagementWrite, @@ -68,7 +69,7 @@ func (a *Analyzer) Analyze(ctx context.Context, credentialInfo map[string]string CallManagementRead, CallManagementWrite, } - } else if info.AccountStatusCode == 401 { + case 401: permissions = []Permission{ ServiceVerificationRead, ServiceVerificationWrite, @@ -159,7 +160,7 @@ func getAccountsStatusCode(cfg *config.Config, sid string, secret string) (int, if err != nil { return 0, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() return resp.StatusCode, nil } @@ -196,7 +197,7 @@ func getVerifyServicesStatusCode(cfg *config.Config, sid string, secret string) if err != nil { return serviceRes, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // read response if err := json.NewDecoder(resp.Body).Decode(&serviceRes); err != nil { @@ -224,7 +225,7 @@ func listTwilioAccounts(cfg *config.Config, sid, secret string) ([]service, erro if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() var result struct { Accounts []service `json:"accounts"` @@ -287,11 +288,12 @@ func printPermissions(statusCode int) { color.Green("[!] Valid Twilio API Key\n") color.Green("[i] Expires: Never") - if statusCode == 401 { + switch statusCode { + case 401: color.Yellow("[i] Key type: Standard") color.Yellow("[i] Permissions: All EXCEPT key management and account/subaccount configuration.") - } else if statusCode == 200 { + case 200: color.Green("[i] Key type: Main (aka Admin)") color.Green("[i] Permissions: All") } diff --git a/pkg/analyzer/generate_permissions/generate_permissions.go b/pkg/analyzer/generate_permissions/generate_permissions.go index c26b61e41f93..62726d0ce72c 100644 --- a/pkg/analyzer/generate_permissions/generate_permissions.go +++ b/pkg/analyzer/generate_permissions/generate_permissions.go @@ -2,9 +2,9 @@ package main import ( "fmt" + regexp "github.com/wasilibs/go-re2" "log" "os" - "regexp" "strings" "text/template" @@ -111,7 +111,7 @@ func main() { if err != nil { log.Fatalf("Failed to open YAML file: %v", err) } - defer file.Close() + defer func() { _ = file.Close() }() var data PermissionsData decoder := yaml.NewDecoder(file) @@ -135,7 +135,7 @@ func main() { if err != nil { log.Fatalf("Failed to create output file: %v", err) } - defer outputFile.Close() + defer func() { _ = outputFile.Close() }() err = tmpl.Execute(outputFile, data) if err != nil { diff --git a/pkg/buffers/buffer/buffer.go b/pkg/buffers/buffer/buffer.go index b3bd17749d6c..b25b9507c5a4 100644 --- a/pkg/buffers/buffer/buffer.go +++ b/pkg/buffers/buffer/buffer.go @@ -47,11 +47,11 @@ func (b *Buffer) Write(data []byte) (int, error) { } size := len(data) - bufferLength := b.Buffer.Len() + bufferLength := b.Len() totalSizeNeeded := bufferLength + size // If the total size is within the threshold, write to the buffer. - availableSpace := b.Buffer.Cap() - bufferLength + availableSpace := b.Cap() - bufferLength growSize := totalSizeNeeded - bufferLength if growSize > availableSpace { // We are manually growing the buffer so we can track the growth via metrics. diff --git a/pkg/cleantemp/cleantemp.go b/pkg/cleantemp/cleantemp.go index 8a9d39e48219..56ff4770ff96 100644 --- a/pkg/cleantemp/cleantemp.go +++ b/pkg/cleantemp/cleantemp.go @@ -2,10 +2,10 @@ package cleantemp import ( "fmt" + regexp "github.com/wasilibs/go-re2" "io" "os" "path/filepath" - "regexp" "strconv" "strings" @@ -73,7 +73,7 @@ func CleanTempArtifacts(ctx logContext.Context) error { if err != nil { return fmt.Errorf("error opening temp dir: %w", err) } - defer dir.Close() + defer func() { _ = dir.Close() }() for { entries, err := dir.ReadDir(1) // read only one entry diff --git a/pkg/common/filter.go b/pkg/common/filter.go index 8462ccac05fb..e274bb6b1e45 100644 --- a/pkg/common/filter.go +++ b/pkg/common/filter.go @@ -3,8 +3,8 @@ package common import ( "bufio" "fmt" + regexp "github.com/wasilibs/go-re2" "os" - "regexp" "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) @@ -14,7 +14,7 @@ type Filter struct { exclude *FilterRuleSet } -type FilterRuleSet []regexp.Regexp +type FilterRuleSet []*regexp.Regexp // FilterEmpty returns a Filter that always passes. func FilterEmpty() *Filter { @@ -39,7 +39,7 @@ func FilterFromFiles(includeFilterPath, excludeFilterPath string) (*Filter, erro // If no includeFilterPath is provided, every pattern should pass the include rules. if includeFilterPath == "" { - includeRules = &FilterRuleSet{*regexp.MustCompile("")} + includeRules = &FilterRuleSet{regexp.MustCompile("")} } filter := &Filter{ @@ -87,7 +87,7 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) { if err != nil { return nil, fmt.Errorf("can not compile regular expression: %s", line) } - rules = append(rules, *pattern) + rules = append(rules, pattern) } return &rules, nil } diff --git a/pkg/common/filter_test.go b/pkg/common/filter_test.go index a4df8db62265..1499f699c053 100644 --- a/pkg/common/filter_test.go +++ b/pkg/common/filter_test.go @@ -1,8 +1,8 @@ package common import ( + regexp "github.com/wasilibs/go-re2" "os" - "regexp" "testing" ) @@ -15,54 +15,54 @@ func TestFilterBasic(t *testing.T) { tests := map[string]filterTest{ "IncludePassed": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("test")}, + include: &FilterRuleSet{regexp.MustCompile("test")}, }, pattern: "teststring", pass: true, }, "IncludeFiltered": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("nomatch")}, + include: &FilterRuleSet{regexp.MustCompile("nomatch")}, }, pattern: "teststring", pass: false, }, "ExcludePassed": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("")}, - exclude: &FilterRuleSet{*regexp.MustCompile("nomatch")}, + include: &FilterRuleSet{regexp.MustCompile("")}, + exclude: &FilterRuleSet{regexp.MustCompile("nomatch")}, }, pattern: "teststring", pass: true, }, "ExcludeFiltered": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("")}, - exclude: &FilterRuleSet{*regexp.MustCompile("test")}, + include: &FilterRuleSet{regexp.MustCompile("")}, + exclude: &FilterRuleSet{regexp.MustCompile("test")}, }, pattern: "teststring", pass: false, }, "IncludeExcludeDifferentPass": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("test")}, - exclude: &FilterRuleSet{*regexp.MustCompile("nomatch")}, + include: &FilterRuleSet{regexp.MustCompile("test")}, + exclude: &FilterRuleSet{regexp.MustCompile("nomatch")}, }, pattern: "teststring", pass: true, }, "IncludeExcludeDifferentFiltered": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("nomatch")}, - exclude: &FilterRuleSet{*regexp.MustCompile("test")}, + include: &FilterRuleSet{regexp.MustCompile("nomatch")}, + exclude: &FilterRuleSet{regexp.MustCompile("test")}, }, pattern: "teststring", pass: false, }, "IncludeExcludeSameFiltered": { filter: Filter{ - include: &FilterRuleSet{*regexp.MustCompile("test")}, - exclude: &FilterRuleSet{*regexp.MustCompile("test")}, + include: &FilterRuleSet{regexp.MustCompile("test")}, + exclude: &FilterRuleSet{regexp.MustCompile("test")}, }, pattern: "teststring", pass: false, @@ -144,14 +144,14 @@ func TestFilterFromFile(t *testing.T) { if err := testFilterWriteFile(includeTestFile, []byte(test.includeFileContents)); err != nil { t.Fatalf("failed to create include rules file: %s", err) } - defer os.Remove(includeTestFile) + defer func() { _ = os.Remove(includeTestFile) }() } if test.excludeFile { excludeTestFile = "/tmp/trufflehog_test_xfilter.txt" if err := testFilterWriteFile(excludeTestFile, []byte(test.excludeFileContents)); err != nil { t.Fatalf("failed to create include rules file: %s", err) } - defer os.Remove(excludeTestFile) + defer func() { _ = os.Remove(excludeTestFile) }() } filter, err := FilterFromFiles(includeTestFile, excludeTestFile) diff --git a/pkg/common/http.go b/pkg/common/http.go index c44b7a70f43b..929b02bedada 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -134,11 +134,8 @@ type InstrumentedTransport struct { } func (t *InstrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { - - sanitizedURL := sanitizeURL(req.URL.String()) - - // increment counter for the URL - recordHTTPRequest(sanitizedURL) + // increment counter for the request + recordHTTPRequest() // Record start time for latency measurement start := time.Now() @@ -149,13 +146,13 @@ func (t *InstrumentedTransport) RoundTrip(req *http.Request) (*http.Response, er duration := time.Since(start) if err != nil { - recordNetworkError(sanitizedURL) + recordNetworkError() return nil, err } if resp != nil { // record latency, response size and increment counter for non-200 status code - recordHTTPResponse(sanitizedURL, resp.StatusCode, duration.Seconds(), resp.ContentLength) + recordHTTPResponse(resp.StatusCode, duration.Seconds(), resp.ContentLength) } return resp, err diff --git a/pkg/common/http_metrics.go b/pkg/common/http_metrics.go index 868112d46e42..3fd5b73f5de2 100644 --- a/pkg/common/http_metrics.go +++ b/pkg/common/http_metrics.go @@ -1,7 +1,6 @@ package common import ( - "net/url" "strconv" "github.com/prometheus/client_golang/prometheus" @@ -9,25 +8,23 @@ import ( ) var ( - httpRequestsTotal = promauto.NewCounterVec( + httpRequestsTotal = promauto.NewCounter( prometheus.CounterOpts{ Namespace: MetricsNamespace, Subsystem: "http_client", Name: "requests_total", - Help: "Total number of HTTP requests made, labeled by URL.", + Help: "Total number of HTTP requests made.", }, - []string{"url"}, ) - httpRequestDuration = promauto.NewHistogramVec( + httpRequestDuration = promauto.NewHistogram( prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: "http_client", Name: "request_duration_seconds", - Help: "HTTP request latency in seconds, labeled by URL.", + Help: "HTTP request latency in seconds.", Buckets: prometheus.DefBuckets, }, - []string{"url"}, ) httpNon200ResponsesTotal = promauto.NewCounterVec( @@ -35,90 +32,44 @@ var ( Namespace: MetricsNamespace, Subsystem: "http_client", Name: "non_200_responses_total", - Help: "Total number of non-200 HTTP responses, labeled by URL and status code.", + Help: "Total number of non-200 HTTP responses, labeled by status code.", }, - []string{"url", "status_code"}, + []string{"status_code"}, ) - httpResponseBodySizeBytes = promauto.NewHistogramVec( + httpResponseBodySizeBytes = promauto.NewHistogram( prometheus.HistogramOpts{ Namespace: MetricsNamespace, Subsystem: "http_client", Name: "response_body_size_bytes", - Help: "Size of HTTP response bodies in bytes, labeled by URL.", + Help: "Size of HTTP response bodies in bytes.", Buckets: prometheus.ExponentialBuckets(100, 10, 5), // [100B, 1KB, 10KB, 100KB, 1MB] }, - []string{"url"}, ) ) -// sanitizeURL sanitizes a URL to avoid high cardinality metrics. -// It keeps only the host and path, removing query parameters, fragments, and user info. -func sanitizeURL(rawURL string) string { - if rawURL == "" { - return "unknown" - } - - parsedURL, err := url.Parse(rawURL) - if err != nil { - return "invalid_url" - } - - // Build sanitized URL with just scheme, host, and path - sanitized := &url.URL{ - Scheme: parsedURL.Scheme, - Host: parsedURL.Host, - Path: parsedURL.Path, - } - - // If host is empty, try to extract from the raw URL - if sanitized.Host == "" { - // For relative URLs or malformed URLs, just use a placeholder - return "relative_or_invalid" - } - - // Normalize path - if sanitized.Path == "" { - sanitized.Path = "/" - } - - // Limit path length to avoid extremely long paths creating high cardinality - if len(sanitized.Path) > 100 { - sanitized.Path = sanitized.Path[:100] + "..." - } - - result := sanitized.String() - - // Final fallback to avoid empty strings - if result == "" { - return "unknown" - } - - return result -} - // recordHTTPRequest records metrics for an HTTP request. -func recordHTTPRequest(sanitizedURL string) { - httpRequestsTotal.WithLabelValues(sanitizedURL).Inc() +func recordHTTPRequest() { + httpRequestsTotal.Inc() } // recordHTTPResponse records metrics for an HTTP response. -func recordHTTPResponse(sanitizedURL string, statusCode int, durationSeconds float64, contentLength int64) { +func recordHTTPResponse(statusCode int, durationSeconds float64, contentLength int64) { // Record latency - httpRequestDuration.WithLabelValues(sanitizedURL).Observe(durationSeconds) + httpRequestDuration.Observe(durationSeconds) // Record non-200 responses if statusCode != 200 { - httpNon200ResponsesTotal.WithLabelValues(sanitizedURL, strconv.Itoa(statusCode)).Inc() + httpNon200ResponsesTotal.WithLabelValues(strconv.Itoa(statusCode)).Inc() } // Record response body size if known if contentLength >= 0 { - httpResponseBodySizeBytes.WithLabelValues(sanitizedURL).Observe(float64(contentLength)) + httpResponseBodySizeBytes.Observe(float64(contentLength)) } } // recordNetworkError records metrics for failed HTTP response -func recordNetworkError(sanitizedURL string) { - httpNon200ResponsesTotal.WithLabelValues(sanitizedURL, "network_error").Inc() +func recordNetworkError() { + httpNon200ResponsesTotal.WithLabelValues("network_error").Inc() } diff --git a/pkg/common/http_test.go b/pkg/common/http_test.go index 8f09a458832b..3b6e34986194 100644 --- a/pkg/common/http_test.go +++ b/pkg/common/http_test.go @@ -95,10 +95,10 @@ func TestCustomTransportSendsCustomHeaders(t *testing.T) { resetCustomHeaders(t) feature.CustomHeaders.Store(http.Header{ - "X-Scanner-Id": []string{"test-scanner"}, - "X-Multi": []string{"one", "two"}, - "User-Agent": []string{"OverrideUA"}, - "X-Empty-Value": []string{""}, + "X-Scanner-Id": []string{"test-scanner"}, + "X-Multi": []string{"one", "two"}, + "User-Agent": []string{"OverrideUA"}, + "X-Empty-Value": []string{""}, }) var captured http.Header @@ -114,7 +114,7 @@ func TestCustomTransportSendsCustomHeaders(t *testing.T) { } resp, err := client.Get(server.URL) require.NoError(t, err) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() assert.Equal(t, "test-scanner", captured.Get("X-Scanner-Id")) assert.Equal(t, []string{"one", "two"}, captured.Values("X-Multi")) @@ -382,62 +382,6 @@ func TestRetryableHTTPClientTimeout(t *testing.T) { } } -func TestSanitizeURL(t *testing.T) { - testCases := []struct { - name string - input string - expected string - }{ - { - name: "valid https URL", - input: "https://api.example.com/v1/users", - expected: "https://api.example.com/v1/users", - }, - { - name: "URL with query parameters", - input: "https://api.example.com/search?q=secret&limit=10", - expected: "https://api.example.com/search", - }, - { - name: "URL with fragment", - input: "https://example.com/page#section", - expected: "https://example.com/page", - }, - { - name: "URL with user info", - input: "https://user:pass@api.example.com/path", - expected: "https://api.example.com/path", - }, - { - name: "empty URL", - input: "", - expected: "unknown", - }, - { - name: "invalid URL", - input: "not-a-url", - expected: "relative_or_invalid", - }, - { - name: "very long path", - input: "https://example.com/" + strings.Repeat("a", 150), - expected: "https://example.com/" + strings.Repeat("a", 99) + "...", // 99 + 1 ("/") = 100 chars - }, - { - name: "root path", - input: "https://example.com", - expected: "https://example.com/", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := sanitizeURL(tc.input) - assert.Equal(t, tc.expected, result) - }) - } -} - func TestSaneHttpClientMetrics(t *testing.T) { // Create a test server that returns different status codes server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -497,18 +441,17 @@ func TestSaneHttpClientMetrics(t *testing.T) { } // Get initial metric values - sanitizedURL := sanitizeURL(requestURL) - initialRequestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL)) + initialRequestsTotal := testutil.ToFloat64(httpRequestsTotal) // Make the request resp, err := client.Get(requestURL) require.NoError(t, err) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() assert.Equal(t, tc.expectedStatusCode, resp.StatusCode) // Check that request counter was incremented - requestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL)) + requestsTotal := testutil.ToFloat64(httpRequestsTotal) assert.Equal(t, initialRequestsTotal+1, requestsTotal) }) } @@ -564,18 +507,17 @@ func TestRetryableHttpClientMetrics(t *testing.T) { } // Get initial metric values - sanitizedURL := sanitizeURL(requestURL) - initialRequestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL)) + initialRequestsTotal := testutil.ToFloat64(httpRequestsTotal) // Make the request resp, err := client.Get(requestURL) require.NoError(t, err) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() assert.Equal(t, tc.expectedStatusCode, resp.StatusCode) // Check that request counter was incremented - requestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL)) + requestsTotal := testutil.ToFloat64(httpRequestsTotal) assert.Equal(t, initialRequestsTotal+1, requestsTotal) }) } @@ -597,19 +539,18 @@ func TestInstrumentedTransport(t *testing.T) { } // Get initial metric value - sanitizedURL := sanitizeURL(server.URL) - initialCount := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL)) + initialCount := testutil.ToFloat64(httpRequestsTotal) // Make a request resp, err := client.Get(server.URL) require.NoError(t, err) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Verify the request was successful assert.Equal(t, http.StatusOK, resp.StatusCode) // Verify metrics were recorded - finalCount := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL)) + finalCount := testutil.ToFloat64(httpRequestsTotal) assert.Equal(t, initialCount+1, finalCount) // Note: Testing histogram metrics is complex due to the way Prometheus handles them diff --git a/pkg/common/patterns.go b/pkg/common/patterns.go index 6ee68b5221f8..812a41395580 100644 --- a/pkg/common/patterns.go +++ b/pkg/common/patterns.go @@ -2,7 +2,7 @@ package common import ( "fmt" - "regexp" + regexp "github.com/wasilibs/go-re2" "strconv" "strings" ) diff --git a/pkg/common/patterns_test.go b/pkg/common/patterns_test.go index a52f517e0459..ce2d2082f381 100644 --- a/pkg/common/patterns_test.go +++ b/pkg/common/patterns_test.go @@ -1,7 +1,7 @@ package common import ( - "regexp" + regexp "github.com/wasilibs/go-re2" "testing" "github.com/stretchr/testify/assert" @@ -95,7 +95,7 @@ func TestPasswordRegexCheck(t *testing.T) { passwordRegexPat := PasswordRegexCheck(passwordPattern) expectedRegexPattern := regexp.MustCompile(passwordRegex) - assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern) + assert.Equal(t, expectedRegexPattern.String(), passwordRegexPat.compiledRegex.String()) testString := `password = "johnsmith123$!" password='johnsmith123$!' diff --git a/pkg/common/secrets.go b/pkg/common/secrets.go index cc759248eed8..0dc444e17cae 100644 --- a/pkg/common/secrets.go +++ b/pkg/common/secrets.go @@ -54,7 +54,7 @@ func GetSecret(ctx context.Context, gcpProject, name string) (secret *Secret, er if err != nil { return nil, errors.Errorf("failed to create secretmanager client: %v", err) } - defer client.Close() + defer func() { _ = client.Close() }() req := &secretmanagerpb.AccessSecretVersionRequest{ Name: parent, diff --git a/pkg/config/config.go b/pkg/config/config.go index 47fe3868c5d8..ef6b2d5a8d2d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources/github" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/jenkins" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/json_enumerator" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/postman" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3" ) @@ -97,6 +98,8 @@ func instantiateSourceFromType(sourceType string) (sources.Source, error) { source = new(filesystem.Source) case sourcespb.SourceType_SOURCE_TYPE_JENKINS.String(): source = new(jenkins.Source) + case sourcespb.SourceType_SOURCE_TYPE_JSON_ENUMERATOR.String(): + source = new(json_enumerator.Source) case sourcespb.SourceType_SOURCE_TYPE_GCS.String(): source = new(gcs.Source) case sourcespb.SourceType_SOURCE_TYPE_GCS_UNAUTHED.String(): diff --git a/pkg/custom_detectors/CUSTOM_DETECTORS.md b/pkg/custom_detectors/CUSTOM_DETECTORS.md index 6bbd3ac6cfd2..f149f81f096b 100644 --- a/pkg/custom_detectors/CUSTOM_DETECTORS.md +++ b/pkg/custom_detectors/CUSTOM_DETECTORS.md @@ -36,6 +36,36 @@ This guide will walk you through setting up a custom detector in TruffleHog to i - **`regex`**: Defines the patterns to identify potential secrets. You can specify one or more named regular expressions. For a detection to be successful, each named regex must find a match. Capture groups `()` within these regular expressions are used to extract specific portions of the matched text, enabling the detector to process and report on particular segments of the identified patterns. - **`verify`**: An optional section to validate detected secrets. If you want to verify or unverify detected secrets, this section needs to be configured. If not configured, all detected secrets will be marked as unverified. Read [verification server examples](#verification-server-examples) + - **`successRanges`**: A list of HTTP status codes (or ranges) that indicate the secret is **live** (active). When the verification server responds with a matching status code, the secret is marked as verified. Each entry can be a single code (`"200"`) or an inclusive range (`"200-202"`). If omitted (along with `rotatedRanges`), only `200` is treated as verified (backward compatible). + - **`rotatedRanges`**: A list of HTTP status codes (or ranges) that indicate the secret has been **rotated** (no longer active). When the verification server responds with a matching status code, the secret is definitively marked as unverified. + + When only one of the two fields is configured, non-matching responses are treated as the opposite state (e.g., if only `successRanges` is set, any response that doesn't match is treated as rotated; if only `rotatedRanges` is set, any non-matching response is treated as live). When both fields are configured and the response matches neither, the result is treated as unknown/inconclusive. + + Since `verify` is optional, Custom Detectors can also be used without a webhook to flag generic hardcoded secrets (e.g. `*.password=`, `*.secret=`) in config files such as `.properties`, `.env`, or `.yaml`. [Here](/examples/generic_config_secrets.yml) is an example config tuned for that use case. + + Here's an example with configurable verification ranges: + + ```yaml + # config.yaml + detectors: + - name: HogTokenDetector + keywords: + - hog + regex: + token: '[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}' + verify: + - endpoint: http://localhost:8000/ + unsafe: true + headers: + - "Authorization: super secret authorization header" + successRanges: + - "200" + rotatedRanges: + - "401" + - "403" + ``` + + In this example, a `200` response from the verification server means the secret is live and needs rotation. A `401` or `403` means the secret has been rotated and is no longer active. Any other response is treated as inconclusive. **Other allowed parameters:** - **`primary_regex_name`**: This parameter allows you designate the primary regex pattern when multiple regex patterns are defined in the regex section. If a match is found, the match for the designated primary regex will be used to determine the line number. The value must be one of the names specified in the regex section. If not provided, the first regex name in sorted order will be used as the primary regex by default. diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.go index 6a9e7a39f3a7..c2683269dce2 100644 --- a/pkg/custom_detectors/custom_detectors.go +++ b/pkg/custom_detectors/custom_detectors.go @@ -4,10 +4,11 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "maps" "net/http" - "regexp" + "regexp" //nolint:depguard // used instead of github.com/wasilibs/go-re2 due to differences in utf-8 handling "slices" "strings" @@ -64,6 +65,12 @@ func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebh if err := ValidateVerifyHeaders(verify.Headers); err != nil { return nil, err } + if err := ValidateVerifyRanges(verify.SuccessRanges); err != nil { + return nil, err + } + if err := ValidateVerifyRanges(verify.RotatedRanges); err != nil { + return nil, err + } } // Ensure primary regex name is set. @@ -275,10 +282,15 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string // disrupt other verification. return nil } - // Try each config until we successfully verify. + + var ( + definitive bool + rangesInEffect bool + ) + + // Try each config until we get a definitive answer. for _, verifyConfig := range c.GetVerify() { if common.IsDone(ctx) { - // TODO: Log we're possibly leaving out results. return ctx.Err() } req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody)) @@ -288,11 +300,13 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string for _, header := range verifyConfig.GetHeaders() { key, value, found := strings.Cut(header, ":") if !found { - // Should be unreachable due to validation. continue } req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r ")) } + if req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", "application/json") + } resp, err := httpClient.Do(req) if err != nil { continue @@ -302,27 +316,58 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string _ = resp.Body.Close() }() - if resp.StatusCode == http.StatusOK { - // mark the result as verified - result.Verified = true + successRanges := verifyConfig.GetSuccessRanges() + rotatedRanges := verifyConfig.GetRotatedRanges() - body, err := io.ReadAll(resp.Body) - if err != nil { - continue + if len(successRanges) == 0 && len(rotatedRanges) == 0 { + // Backward compat: no ranges configured, use legacy behavior. + if resp.StatusCode == http.StatusOK { + result.Verified = true + definitive = true + storeResponseBody(resp, result.ExtraData) + break } + // Legacy non-200 is a meaningful response (verifier said "no"); + // mark definitive so a prior ranged verifier with rangesInEffect + // does not cause a spurious verification error. + definitive = true + continue + } - // TODO: handle different content-type responses seperatly when implement custom detector configurations - responseStr := string(body) - // truncate to 200 characters if response length exceeds 200 - if len(responseStr) > 200 { - responseStr = responseStr[:200] - } + rangesInEffect = true + bothConfigured := len(successRanges) > 0 && len(rotatedRanges) > 0 + + if StatusCodeMatchesRanges(resp.StatusCode, successRanges) { + result.Verified = true + definitive = true + storeResponseBody(resp, result.ExtraData) + break + } - // store the processed response in ExtraData - result.ExtraData["response"] = responseStr + if StatusCodeMatchesRanges(resp.StatusCode, rotatedRanges) { + definitive = true + break + } + // Status matched neither configured range. + if !bothConfigured { + // Only one side was configured: the non-matching response is + // treated as the opposite state. + // successRanges only -> non-match means rotated + // rotatedRanges only -> non-match means live + definitive = true + if len(rotatedRanges) > 0 { + result.Verified = true + storeResponseBody(resp, result.ExtraData) + } break } + + // Both configured but neither matched -- try the next verifier. + } + + if rangesInEffect && !definitive { + result.SetVerificationError(errors.New("verification response status code did not match any configured successRanges or rotatedRanges")) } select { @@ -333,6 +378,20 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string } } +const maxResponseLen = 200 + +func storeResponseBody(resp *http.Response, extraData map[string]string) { + body, err := io.ReadAll(resp.Body) + if err != nil { + return + } + responseStr := string(body) + if len(responseStr) > maxResponseLen { + responseStr = responseStr[:maxResponseLen] + } + extraData["response"] = responseStr +} + func (c *CustomRegexWebhook) Keywords() []string { return c.GetKeywords() } diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go index c3a97a461e64..760971b68e3c 100644 --- a/pkg/custom_detectors/custom_detectors_test.go +++ b/pkg/custom_detectors/custom_detectors_test.go @@ -2,6 +2,8 @@ package custom_detectors import ( "context" + "net/http" + "net/http/httptest" "strings" "testing" @@ -47,6 +49,30 @@ verify: assert.Equal(t, []string{"200-250", "288"}, got.Verify[0].SuccessRanges) } +func TestCustomRegexTemplateParsingWithRotatedRanges(t *testing.T) { + testYaml := `name: test +keywords: +- secret +regex: + token: ([a-zA-Z0-9]{32}) +verify: +- endpoint: http://localhost:8000/ + unsafe: true + headers: + - 'Authorization: Bearer token' + successRanges: + - '200' + rotatedRanges: + - '401' + - 403-404` + + var got custom_detectorspb.CustomRegex + assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testYaml), &got)) + assert.Equal(t, 1, len(got.Verify)) + assert.Equal(t, []string{"200"}, got.Verify[0].SuccessRanges) + assert.Equal(t, []string{"401", "403-404"}, got.Verify[0].RotatedRanges) +} + func TestCustomRegexWebhookParsing(t *testing.T) { testCustomRegexWebhookYaml := `name: Internal bi tool keywords: @@ -606,7 +632,10 @@ func TestDetectorValidations(t *testing.T) { results, err := detector.FromData(context.Background(), false, []byte(tt.input.Data)) assert.NoError(t, err) - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "ExtraData", "verificationError", "primarySecret") + ignoreOpts := cmp.Options{ + cmpopts.IgnoreUnexported(detectors.Result{}), + cmpopts.IgnoreFields(detectors.Result{}, "ExtraData"), + } if diff := cmp.Diff(results, tt.want, ignoreOpts); diff != "" { t.Errorf("CustomDetector.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } @@ -738,6 +767,26 @@ func TestNewWebhookCustomRegex_Validation(t *testing.T) { wantErr: true, wantErrSubstr: `must contain a colon`, }, + { + name: "ValidateVerifyRanges: invalid successRanges", + mutate: func(pb *custom_detectorspb.CustomRegex) { + pb.Verify = []*custom_detectorspb.VerifierConfig{ + {Endpoint: "https://example.com/verify", Headers: []string{"A: b"}, SuccessRanges: []string{"abc"}}, + } + }, + wantErr: true, + wantErrSubstr: "unable to convert http code to int", + }, + { + name: "ValidateVerifyRanges: invalid rotatedRanges", + mutate: func(pb *custom_detectorspb.CustomRegex) { + pb.Verify = []*custom_detectorspb.VerifierConfig{ + {Endpoint: "https://example.com/verify", Headers: []string{"A: b"}, RotatedRanges: []string{"999"}}, + } + }, + wantErr: true, + wantErrSubstr: "invalid http status code", + }, } for _, tt := range tests { @@ -779,6 +828,167 @@ func TestNewWebhookCustomRegex_EnsurePrimaryRegexNameSet(t *testing.T) { assert.Equal(t, "regex_a", detector.GetPrimaryRegexName(), "expected PrimaryRegexName to be set to regex_a") } +func TestVerificationWithConfigurableRanges(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serverStatus int + successRanges []string + rotatedRanges []string + wantVerified bool + wantVerifyErr bool + }{ + { + name: "backward compat: no ranges, 200 -> verified", + serverStatus: 200, + wantVerified: true, + wantVerifyErr: false, + }, + { + name: "backward compat: no ranges, 401 -> unverified", + serverStatus: 401, + wantVerified: false, + wantVerifyErr: false, + }, + { + name: "successRanges match -> verified", + serverStatus: 201, + successRanges: []string{"200-202"}, + wantVerified: true, + wantVerifyErr: false, + }, + { + name: "rotatedRanges match -> not verified, no error", + serverStatus: 401, + successRanges: []string{"200"}, + rotatedRanges: []string{"401", "403"}, + wantVerified: false, + wantVerifyErr: false, + }, + { + name: "neither match -> not verified, verification error", + serverStatus: 500, + successRanges: []string{"200"}, + rotatedRanges: []string{"401"}, + wantVerified: false, + wantVerifyErr: true, + }, + { + name: "successRanges range boundary inclusive", + serverStatus: 250, + successRanges: []string{"200-250"}, + wantVerified: true, + wantVerifyErr: false, + }, + { + name: "only successRanges: non-match means rotated", + serverStatus: 401, + successRanges: []string{"200"}, + wantVerified: false, + wantVerifyErr: false, + }, + { + name: "only rotatedRanges: non-match means live", + serverStatus: 200, + rotatedRanges: []string{"401", "403"}, + wantVerified: true, + wantVerifyErr: false, + }, + { + name: "only rotatedRanges: match means rotated", + serverStatus: 401, + rotatedRanges: []string{"401"}, + wantVerified: false, + wantVerifyErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.serverStatus) + _, _ = w.Write([]byte(`{"status":"ok"}`)) + })) + defer ts.Close() + + detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{ + Name: "test", + Keywords: []string{"secret"}, + Regex: map[string]string{"token": `(secret_[a-zA-Z0-9]{10})`}, + Verify: []*custom_detectorspb.VerifierConfig{ + { + Endpoint: ts.URL, + Unsafe: true, + Headers: []string{"Authorization: Bearer test"}, + SuccessRanges: tt.successRanges, + RotatedRanges: tt.rotatedRanges, + }, + }, + }) + assert.NoError(t, err) + + results, err := detector.FromData(context.Background(), true, []byte("secret_ABCDEFGHIJ")) + assert.NoError(t, err) + assert.Equal(t, 1, len(results), "expected exactly one result") + + result := results[0] + assert.Equal(t, tt.wantVerified, result.Verified, "Verified mismatch") + if tt.wantVerifyErr { + assert.NotNil(t, result.VerificationError(), "expected a verification error") + } else { + assert.Nil(t, result.VerificationError(), "expected no verification error") + } + }) + } +} + +func TestVerificationMixedRangedAndLegacyVerifiers(t *testing.T) { + t.Parallel() + + // Verifier 1 has ranges configured but returns a status matching neither. + // Verifier 2 is legacy (no ranges) and returns 200. + // The result should be Verified=true with NO verification error. + tsRanged := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer tsRanged.Close() + + tsLegacy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte(`ok`)) + })) + defer tsLegacy.Close() + + detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{ + Name: "test", + Keywords: []string{"secret"}, + Regex: map[string]string{"token": `(secret_[a-zA-Z0-9]{10})`}, + Verify: []*custom_detectorspb.VerifierConfig{ + { + Endpoint: tsRanged.URL, + Unsafe: true, + Headers: []string{"A: b"}, + SuccessRanges: []string{"200"}, + RotatedRanges: []string{"401"}, + }, + { + Endpoint: tsLegacy.URL, + Unsafe: true, + Headers: []string{"A: b"}, + }, + }, + }) + assert.NoError(t, err) + + results, err := detector.FromData(context.Background(), true, []byte("secret_ABCDEFGHIJ")) + assert.NoError(t, err) + assert.Equal(t, 1, len(results)) + assert.True(t, results[0].Verified, "expected Verified=true from legacy fallback") + assert.Nil(t, results[0].VerificationError(), "legacy success must not produce a spurious verification error") +} + func BenchmarkProductIndices(b *testing.B) { for i := 0; i < b.N; i++ { _ = productIndices(3, 2, 6) diff --git a/pkg/custom_detectors/regex_varstring.go b/pkg/custom_detectors/regex_varstring.go index e0efc1364183..79812cd3c4ae 100644 --- a/pkg/custom_detectors/regex_varstring.go +++ b/pkg/custom_detectors/regex_varstring.go @@ -1,7 +1,7 @@ package custom_detectors import ( - "regexp" + regexp "github.com/wasilibs/go-re2" "strconv" "strings" ) diff --git a/pkg/custom_detectors/validation.go b/pkg/custom_detectors/validation.go index b9d3c2f673bf..cb81ae5c426b 100644 --- a/pkg/custom_detectors/validation.go +++ b/pkg/custom_detectors/validation.go @@ -2,7 +2,7 @@ package custom_detectors import ( "fmt" - "regexp" + "regexp" //nolint:depguard // used instead of github.com/wasilibs/go-re2 due to differences in utf-8 handling "strconv" "strings" ) @@ -72,6 +72,28 @@ func ValidateVerifyHeaders(headers []string) error { return nil } +// StatusCodeMatchesRanges reports whether code falls within any of the given +// ranges. Each element is either a single HTTP status code ("200") or a +// hyphenated inclusive range ("200-299"). Entries must have been pre-validated +// by ValidateVerifyRanges; malformed entries are silently skipped. +func StatusCodeMatchesRanges(code int, ranges []string) bool { + for _, r := range ranges { + if !strings.Contains(r, "-") { + if c, err := strconv.Atoi(r); err == nil && c == code { + return true + } + continue + } + parts := strings.SplitN(r, "-", 2) + lo, err1 := strconv.Atoi(parts[0]) + hi, err2 := strconv.Atoi(parts[1]) + if err1 == nil && err2 == nil && code >= lo && code <= hi { + return true + } + } + return false +} + func ValidateVerifyRanges(ranges []string) error { const httpLowerRange = 100 const httpUpperRange = 599 diff --git a/pkg/custom_detectors/validation_test.go b/pkg/custom_detectors/validation_test.go index 39d74cdd81b6..f47bb8905915 100644 --- a/pkg/custom_detectors/validation_test.go +++ b/pkg/custom_detectors/validation_test.go @@ -198,6 +198,90 @@ func TestCustomDetectorsVerifyRangeValidation(t *testing.T) { } } +func TestStatusCodeMatchesRanges(t *testing.T) { + tests := []struct { + name string + code int + ranges []string + want bool + }{ + { + name: "single code match", + code: 200, + ranges: []string{"200"}, + want: true, + }, + { + name: "single code no match", + code: 201, + ranges: []string{"200"}, + want: false, + }, + { + name: "range lower bound", + code: 200, + ranges: []string{"200-250"}, + want: true, + }, + { + name: "range upper bound", + code: 250, + ranges: []string{"200-250"}, + want: true, + }, + { + name: "range middle", + code: 225, + ranges: []string{"200-250"}, + want: true, + }, + { + name: "range outside below", + code: 199, + ranges: []string{"200-250"}, + want: false, + }, + { + name: "range outside above", + code: 251, + ranges: []string{"200-250"}, + want: false, + }, + { + name: "multiple ranges first match", + code: 200, + ranges: []string{"200", "300-399"}, + want: true, + }, + { + name: "multiple ranges second match", + code: 301, + ranges: []string{"200", "300-399"}, + want: true, + }, + { + name: "multiple ranges no match", + code: 500, + ranges: []string{"200", "300-399"}, + want: false, + }, + { + name: "empty ranges", + code: 200, + ranges: nil, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := StatusCodeMatchesRanges(tt.code, tt.ranges) + if got != tt.want { + t.Errorf("StatusCodeMatchesRanges(%d, %v) = %v, want %v", tt.code, tt.ranges, got, tt.want) + } + }) + } +} + func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) { tests := []struct { name string diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.go index 074373f96397..347936eab1c0 100644 --- a/pkg/decoders/escaped_unicode.go +++ b/pkg/decoders/escaped_unicode.go @@ -2,7 +2,7 @@ package decoders import ( "bytes" - "regexp" + regexp "github.com/wasilibs/go-re2" "strconv" "unicode/utf8" @@ -19,7 +19,14 @@ var _ Decoder = (*EscapedUnicode)(nil) var ( // Standard Unicode notation. //https://unicode.org/standard/principles.html - codePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`) + // + // The word boundary that belongs at the front of this pattern is applied by + // codePointMatches instead of being written here. A leading empty-width + // assertion stops regexp from extracting a literal prefix, and without a + // prefix every offset in the chunk has to be tried rather than skipping + // straight to the next "U+". That one difference made this pattern cost + // more than every other pattern in this file combined. + codePointPat = regexp.MustCompile(`U\+([a-fA-F0-9]{4}).?`) // Common escape sequence used in programming languages. escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`) @@ -52,6 +59,53 @@ var ( // hexEscapePat = regexp.MustCompile(`0x([a-fA-F0-9]{1,6})(?:\s|$)`) ) +// Every escape form this decoder understands starts with a fixed literal: a +// backslash for the \uXXXX family and CSS escapes, "U+" for standard code +// point notation, "&#x" for HTML and "%u" for percent encoding. Looking for +// that literal is far cheaper than running the pattern, and a chunk without it +// cannot match, so each pattern is only run once its anchor is present. +var ( + codePointAnchor = []byte("U+") + htmlAnchor = []byte("&#x") + percentAnchor = []byte("%u") +) + +// isWordByte reports whether c is a word character as \b defines it: ASCII +// letters, digits and underscore. +func isWordByte(c byte) bool { + return c == '_' || + ('0' <= c && c <= '9') || + ('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') +} + +// codePointMatches returns the submatch indices of every U+XXXX sequence in +// input that starts on a word boundary. +// +// This is the check the leading \b used to perform. "U" is itself a word +// character, so the boundary holds exactly when the sequence starts the input +// or follows a byte that is not a word character. +// +// Dropping a match here can never hide another one. A second "U+" cannot begin +// inside the four hex digits, since "+" is not a hex digit, so the only overlap +// possible is the trailing optional byte covering a following "U". That "U" +// is preceded by a hex digit, which is a word character, so the boundary would +// have rejected it as well. +func codePointMatches(input []byte) [][]int { + if !bytes.Contains(input, codePointAnchor) { + return nil + } + all := codePointPat.FindAllSubmatchIndex(input, -1) + kept := all[:0] + for _, m := range all { + if m[0] > 0 && isWordByte(input[m[0]-1]) { + continue + } + kept = append(kept, m) + } + return kept +} + func (d *EscapedUnicode) Type() detectorspb.DecoderType { return detectorspb.DecoderType_ESCAPED_UNICODE } @@ -70,29 +124,31 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk { // Process patterns in priority order - more specific patterns first // This prevents conflicts where multiple patterns match the same input + hasBackslash := bytes.IndexByte(chunkData, '\\') >= 0 + // Long escape format (8 hex digits) - highest priority - if longEscapePat.Match(chunkData) { + if hasBackslash && longEscapePat.Match(chunkData) { matched = true chunkData = decodeLongEscape(chunkData) - } else if braceEscapePat.Match(chunkData) { + } else if hasBackslash && braceEscapePat.Match(chunkData) { matched = true chunkData = decodeBraceEscape(chunkData) - } else if perlEscapePat.Match(chunkData) { + } else if hasBackslash && perlEscapePat.Match(chunkData) { matched = true chunkData = decodePerlEscape(chunkData) - } else if htmlEscapePat.Match(chunkData) { + } else if bytes.Contains(chunkData, htmlAnchor) && htmlEscapePat.Match(chunkData) { matched = true chunkData = decodeHtmlEscape(chunkData) - } else if percentEscapePat.Match(chunkData) { + } else if bytes.Contains(chunkData, percentAnchor) && percentEscapePat.Match(chunkData) { matched = true chunkData = decodePercentEscape(chunkData) - } else if escapePat.Match(chunkData) { + } else if hasBackslash && escapePat.Match(chunkData) { matched = true chunkData = decodeEscaped(chunkData) - } else if codePointPat.Match(chunkData) { + } else if codePointIdx := codePointMatches(chunkData); len(codePointIdx) > 0 { matched = true - chunkData = decodeCodePoint(chunkData) - } else if cssEscapePat.Match(chunkData) { + chunkData = decodeCodePointAt(chunkData, codePointIdx) + } else if hasBackslash && cssEscapePat.Match(chunkData) { matched = true chunkData = decodeCssEscape(chunkData) // } else if hexEscapePat.Match(chunkData) { @@ -154,8 +210,13 @@ func decodeWithPattern(input []byte, re *regexp.Regexp) []byte { func decodeCodePoint(input []byte) []byte { // Find all Unicode escape sequences in the input byte slice - indices := codePointPat.FindAllSubmatchIndex(input, -1) + return decodeCodePointAt(input, codePointMatches(input)) +} +// decodeCodePointAt is decodeCodePoint driven by indices the caller already +// has. FromChunk needs the match list to decide whether this pattern applies, +// so it passes that list straight through rather than scanning a second time. +func decodeCodePointAt(input []byte, indices [][]int) []byte { // Iterate over found indices in reverse order to avoid modifying the slice length utf8Bytes := make([]byte, maxBytesPerRune) for i := len(indices) - 1; i >= 0; i-- { diff --git a/pkg/decoders/escaped_unicode_equivalence_test.go b/pkg/decoders/escaped_unicode_equivalence_test.go new file mode 100644 index 000000000000..167624a2622f --- /dev/null +++ b/pkg/decoders/escaped_unicode_equivalence_test.go @@ -0,0 +1,181 @@ +package decoders + +import ( + "bytes" + "math/rand" + "strconv" + "testing" + "unicode/utf8" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// legacyCodePointPat is main's pattern, word boundary and all. +var legacyCodePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`) + +// legacyDecodeCodePoint is main's decodeCodePoint verbatim, driven by the +// boundary-carrying pattern. +func legacyDecodeCodePoint(input []byte) []byte { + indices := legacyCodePointPat.FindAllSubmatchIndex(input, -1) + utf8Bytes := make([]byte, maxBytesPerRune) + for i := len(indices) - 1; i >= 0; i-- { + matches := indices[i] + startIndex, endIndex := matches[0], matches[1] + hexStartIndex, hexEndIndex := matches[2], matches[3] + if endIndex != hexEndIndex && input[endIndex-1] != spaceChar { + endIndex = endIndex - 1 + } + unicodeInt, err := strconv.ParseInt(string(input[hexStartIndex:hexEndIndex]), 16, 32) + if err != nil { + continue + } + utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt)) + input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...) + } + return input +} + +// compareIndices checks that the boundary filter selects exactly the matches +// the \b pattern would have found. +func compareIndices(t *testing.T, data []byte) { + t.Helper() + want := legacyCodePointPat.FindAllSubmatchIndex(data, -1) + got := codePointMatches(bytes.Clone(data)) + if len(want) != len(got) { + t.Fatalf("input %q: legacy found %d matches %v, filtered found %d %v", data, len(want), want, len(got), got) + } + for i := range want { + for j := range want[i] { + if want[i][j] != got[i][j] { + t.Fatalf("input %q: match %d differs: legacy %v filtered %v", data, i, want[i], got[i]) + } + } + } +} + +// compareDecode checks the full decode output agrees. +func compareDecode(t *testing.T, data []byte) { + t.Helper() + want := legacyDecodeCodePoint(bytes.Clone(data)) + got := decodeCodePoint(bytes.Clone(data)) + if !bytes.Equal(want, got) { + t.Fatalf("input %q: legacy decoded to %q, new decoded to %q", data, want, got) + } +} + +// compareMatched checks the decoder's decode/skip decision agrees. +func compareMatched(t *testing.T, data []byte) { + t.Helper() + wantMatched := legacyCodePointPat.Match(data) + gotMatched := len(codePointMatches(bytes.Clone(data))) > 0 + if wantMatched != gotMatched { + t.Fatalf("input %q: legacy matched=%v, new matched=%v", data, wantMatched, gotMatched) + } +} + +var codePointSeeds = []string{ + "", "U+0041", "U+0041 ", "U+0041x", "U+0041U+0042", "U+0041 U+0042", + "AU+0041", "_U+0041", "9U+0041", " U+0041", "\tU+0041", "\nU+0041", + "AU+0041 U+0042", "U+0041\nU+0042", "U+0041\tU+0042", "U+00", + "U+ZZZZ", "u+0041", "UU+0041", "U+0041U", "U+0041+", "xU+0041U+0042 U+0043", + "U+D800", "U+FFFF", "U+0000", "café U+00E9", "\xff\xfeU+0041", + "U+0041U+0042U+0043", "-U+0041-U+0042-", "U+0041.U+0042", "1U+0041 2U+0042", +} + +func TestCodePointBoundaryEquivalence(t *testing.T) { + for _, s := range codePointSeeds { + compareIndices(t, []byte(s)) + compareDecode(t, []byte(s)) + compareMatched(t, []byte(s)) + } + + // Alphabet weighted towards everything the pattern and boundary care about. + alphabet := []byte("UU++0123456789abcdefABCDEFxyz_ \t\n.-\xff") + rnd := rand.New(rand.NewSource(7)) + for i := 0; i < 300000; i++ { + b := make([]byte, 1+rnd.Intn(40)) + for j := range b { + b[j] = alphabet[rnd.Intn(len(alphabet))] + } + compareIndices(t, b) + compareDecode(t, b) + compareMatched(t, b) + } +} + +// legacyFromChunk is main's whole FromChunk, unanchored and with \b. +func legacyFromChunk(chunk *sources.Chunk) []byte { + chunkData := bytes.Clone(chunk.Data) + matched := false + if longEscapePat.Match(chunkData) { + matched, chunkData = true, decodeLongEscape(chunkData) + } else if braceEscapePat.Match(chunkData) { + matched, chunkData = true, decodeBraceEscape(chunkData) + } else if perlEscapePat.Match(chunkData) { + matched, chunkData = true, decodePerlEscape(chunkData) + } else if htmlEscapePat.Match(chunkData) { + matched, chunkData = true, decodeHtmlEscape(chunkData) + } else if percentEscapePat.Match(chunkData) { + matched, chunkData = true, decodePercentEscape(chunkData) + } else if escapePat.Match(chunkData) { + matched, chunkData = true, decodeEscaped(chunkData) + } else if legacyCodePointPat.Match(chunkData) { + matched, chunkData = true, legacyDecodeCodePoint(chunkData) + } else if cssEscapePat.Match(chunkData) { + matched, chunkData = true, decodeCssEscape(chunkData) + } + if !matched { + return nil + } + return chunkData +} + +func compareFromChunk(t *testing.T, data []byte) { + t.Helper() + want := legacyFromChunk(&sources.Chunk{Data: bytes.Clone(data)}) + got := (&EscapedUnicode{}).FromChunk(&sources.Chunk{Data: bytes.Clone(data)}) + switch { + case want == nil && got != nil: + t.Fatalf("new decoded %q to %q, legacy skipped it", data, got.Data) + case want != nil && got == nil: + t.Fatalf("new skipped %q, legacy decoded it to %q", data, want) + case want != nil && !bytes.Equal(want, got.Data): + t.Fatalf("input %q: legacy %q, new %q", data, want, got.Data) + } +} + +func TestFromChunkEquivalence(t *testing.T) { + for _, s := range append(codePointSeeds, + "\\u0041", "\\\\u0041", "\\U0001F600", "\\u{1F600}", "\\x{263A}", + "A", "%u0041", "\\41 ", "\\41\\", "plain text", "a+b&c%d\\e", + ) { + compareFromChunk(t, []byte(s)) + } + alphabet := []byte(`UU++\\\uUxX{}&#%;0123456789abcdefABCDEF _.-` + "\t\n") + rnd := rand.New(rand.NewSource(11)) + for i := 0; i < 300000; i++ { + b := make([]byte, 1+rnd.Intn(48)) + for j := range b { + b[j] = alphabet[rnd.Intn(len(alphabet))] + } + compareFromChunk(t, b) + } +} + +func FuzzCodePointBoundary(f *testing.F) { + for _, s := range codePointSeeds { + f.Add([]byte(s)) + } + f.Add([]byte("\\u0041")) + f.Add([]byte("A")) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) == 0 { + return + } + compareIndices(t, data) + compareDecode(t, data) + compareFromChunk(t, data) + }) +} diff --git a/pkg/decoders/html.go b/pkg/decoders/html.go index 93e7366bd1c9..6a086cca577f 100644 --- a/pkg/decoders/html.go +++ b/pkg/decoders/html.go @@ -2,8 +2,8 @@ package decoders import ( "bytes" + regexp "github.com/wasilibs/go-re2" "net/url" - "regexp" "strings" "golang.org/x/net/html" @@ -23,18 +23,30 @@ func (d *HTML) Type() detectorspb.DecoderType { return detectorspb.DecoderType_HTML } -var htmlTagPattern = regexp.MustCompile(`<[a-zA-Z][a-zA-Z0-9]*[\s>/]`) +// htmlTagPattern matches standard HTML/XML opening tags. The optional namespace +// group (?::[a-zA-Z][a-zA-Z0-9]*)? also matches ASPX/XML namespace-prefixed +// tags such as /]`) + +// htmlEntityPattern matches entity-encoded HTML tags (e.g. <div , <p>, +// <br/>). The terminator (?:[\s>/]|>) covers both literal > and its +// entity form, so bare-tag forms like <p> are detected. Requiring a +// terminator after the tag name prevents matching comparison operators in +// entity-encoded text (x < threshold has a space before the word, not +// after) and template placeholders (<YOUR_KEY> where _ breaks the +// alphanumeric run before > can match). +var htmlEntityPattern = regexp.MustCompile(`<[a-zA-Z][a-zA-Z0-9]*(?::[a-zA-Z][a-zA-Z0-9]*)?(?:[\s>/]|>)`) // highSignalAttrs are attribute names whose values are extracted into the // decoded output because they commonly contain URLs, tokens, or other secrets. var highSignalAttrs = map[string]bool{ - "href": true, - "src": true, - "action": true, - "value": true, - "content": true, - "alt": true, - "title": true, + "href": true, + "src": true, + "action": true, + "value": true, + "content": true, + "alt": true, + "title": true, } // syntaxHighlightPrefixes lists CSS class prefixes used by syntax highlighting @@ -116,7 +128,7 @@ func (d *HTML) FromChunk(chunk *sources.Chunk) *DecodableChunk { } func looksLikeHTML(data []byte) bool { - return htmlTagPattern.Match(data) + return htmlTagPattern.Match(data) || htmlEntityPattern.Match(data) } func extractHTML(data []byte) []byte { @@ -196,9 +208,18 @@ func hasSyntaxHighlightClass(n *html.Node) bool { } func emitAttributes(buf *bytes.Buffer, n *html.Node) { + // Namespace-prefixed elements (e.g. asp:textbox, mso:canvascontent1) are + // ASP.NET server controls or XML metadata nodes. All their attributes are + // data payloads that may carry secrets (ConnectionString, Text, SelectCommand, + // etc.), so we emit every attribute rather than filtering by highSignalAttrs. + // After html.Parse the colon is preserved in n.Data even though the name is + // lowercased, making strings.Contains a reliable namespace check. + isNamespaced := strings.Contains(n.Data, ":") + for _, attr := range n.Attr { - isDataAttr := strings.HasPrefix(attr.Key, "data-") - if !highSignalAttrs[attr.Key] && !isDataAttr { + if !isNamespaced && + !highSignalAttrs[attr.Key] && + !strings.HasPrefix(attr.Key, "data-") { continue } val := strings.TrimSpace(attr.Val) diff --git a/pkg/decoders/html_test.go b/pkg/decoders/html_test.go index 403ebdb2f93a..2a9f03da703e 100644 --- a/pkg/decoders/html_test.go +++ b/pkg/decoders/html_test.go @@ -208,7 +208,7 @@ func TestHTML_FromChunk(t *testing.T) { { // Block elements (

) produce newline boundaries so adjacent // paragraphs don't merge tokens. - name: "block elements produce newlines", + name: "block elements produce newlines", chunk: &sources.Chunk{Data: []byte(`

first

second

`)}, want: "first\nsecond", }, @@ -336,39 +336,106 @@ func TestHTML_FromChunk(t *testing.T) { { // Zero-width spaces inserted between characters by rich text editors // are stripped so detector regexes can match the full token. - name: "zero-width space stripped from secret", + name: "zero-width space stripped from secret", chunk: &sources.Chunk{Data: []byte("

TOKEN_\u200BABCDEF_1234

")}, - want: "TOKEN_ABCDEF_1234", + want: "TOKEN_ABCDEF_1234", }, { // Multiple invisible codepoint types mixed into a single token. - name: "multiple invisible character types stripped", + name: "multiple invisible character types stripped", chunk: &sources.Chunk{Data: []byte("

SECRET\u200C_VALUE\u00AD_HERE\u2060_END\uFEFF

")}, - want: "SECRET_VALUE_HERE_END", + want: "SECRET_VALUE_HERE_END", }, // --- SVG xlink:href attribute extraction --- { // SVG elements use xlink:href for URLs which may contain tokens. - name: "xlink:href extracted from SVG element", + name: "xlink:href extracted from SVG element", chunk: &sources.Chunk{Data: []byte(`icon`)}, - want: "https://api.example.com?token=secret_value_123\nicon", + want: "https://api.example.com?token=secret_value_123\nicon", }, // --- Double-encoded HTML entity decoding --- { // Content double-encoded as &amp; becomes & after the parser's // first pass; the residual entity replacer decodes it to &. - name: "double-encoded ampersand decoded", + name: "double-encoded ampersand decoded", chunk: &sources.Chunk{Data: []byte(`

key=abc&amp;secret=val

`)}, - want: "key=abc&secret=val", + want: "key=abc&secret=val", }, { // Single-encoded entities are handled by the parser; verify the // residual replacer does not corrupt already-decoded content. - name: "single-encoded entities not double-decoded", + name: "single-encoded entities not double-decoded", chunk: &sources.Chunk{Data: []byte(`

5 > 3 & 2 < 4

`)}, - want: "5 > 3 & 2 < 4", + want: "5 > 3 & 2 < 4", + }, + + // --- ASPX / namespace-prefixed tag support --- + { + // Fragment ASPX pages (master-page-based) have no wrapper; + // they start with an <%@ Page %> directive and use + // as their root. The updated htmlTagPattern must match the + // namespace-prefixed tag so the decoder doesn't return nil. + // The <%@ ... %> directive is emitted as text by the HTML5 parser + // (< followed by % is not a valid tag start, so < is treated literally). + // Namespace element attributes (PlaceHolderMain, server) are also + // emitted because is a namespace-prefixed element. + name: "aspx fragment with namespace tags and no html wrapper", + chunk: &sources.Chunk{Data: []byte( + "<%@ Page MasterPageFile=\"~masterurl/default.master\" %>\n" + + `` + + "
api_key=abc123
" + + "", + )}, + want: "<%@ Page MasterPageFile=\"~masterurl/default.master\" %>\nPlaceHolderMain\nserver\napi_key=abc123", + }, + { + // ASP.NET server controls carry secrets in PascalCase attributes + // (ConnectionString, Text, SelectCommand, etc.) that are not in + // highSignalAttrs and do not use the data- prefix. Namespace-prefixed + // elements must have all their attributes emitted. + name: "aspx server control attribute extraction - ConnectionString", + chunk: &sources.Chunk{Data: []byte(``)}, + want: "Server=.;Password=hunter2\nserver", + }, + { + // A plain Text attribute on a server control (common for labels, + // textboxes, buttons that carry pre-filled values). + name: "aspx server control attribute extraction - Text", + chunk: &sources.Chunk{Data: []byte(``)}, + want: "api_key=secret123\nserver", + }, + { + // Entity-encoded HTML chunk with no literal tags. This simulates a + // chunk that falls entirely within a large entity-encoded field (e.g. + // SharePoint mso:CanvasContent1). The htmlEntityPattern must fire so + // the decoder does not return nil. html.Parse decodes the entities to + // literal HTML markup — the engine's iterativeDecode then re-applies + // the HTML decoder at depth 2 to fully extract the secret. At depth 1 + // the output is the decoded markup string. + name: "entity-encoded html chunk with no literal tags", + chunk: &sources.Chunk{Data: []byte(`<div data-sp-rte=""><p>api_key=secret123</p></div>`)}, + want: `

api_key=secret123

`, + }, + { + // SharePoint stores page content entity-encoded inside mso:CanvasContent1. + // The first html.Parse pass decodes the entities to a text string containing + // literal HTML markup. The engine's iterativeDecode then re-applies the HTML + // decoder at depth 2 to extract content from that inner markup. + // At depth 1 (this test), the entity-encoded text is emitted as plain text + // after residualEntityReplacer runs. The msdt:dt="string" attribute is also + // emitted because mso:CanvasContent1 is a namespace-prefixed element. + name: "sharepoint aspx mso:CanvasContent1 secret in nested html", + chunk: &sources.Chunk{Data: []byte( + `` + + `` + + `` + + `<div><p>-----BEGIN OPENSSH PRIVATE KEY-----</p></div>` + + `` + + ``, + )}, + want: "string\n

-----BEGIN OPENSSH PRIVATE KEY-----

", }, // --- Integration: all extraction types in one chunk --- @@ -398,7 +465,7 @@ func TestHTML_FromChunk(t *testing.T) { if tt.wantNil { if got != nil { - t.Errorf("FromChunk() = %q, want nil", string(got.Chunk.Data)) + t.Errorf("FromChunk() = %q, want nil", string(got.Data)) } return } @@ -409,8 +476,8 @@ func TestHTML_FromChunk(t *testing.T) { if got.DecoderType != detectorspb.DecoderType_HTML { t.Errorf("DecoderType = %v, want %v", got.DecoderType, detectorspb.DecoderType_HTML) } - if string(got.Chunk.Data) != tt.want { - t.Errorf("FromChunk() data =\n%q\nwant:\n%q", string(got.Chunk.Data), tt.want) + if string(got.Data) != tt.want { + t.Errorf("FromChunk() data =\n%q\nwant:\n%q", string(got.Data), tt.want) } }) } @@ -423,7 +490,7 @@ func TestHTML_FeatureFlagDisabled(t *testing.T) { d := &HTML{} chunk := &sources.Chunk{Data: []byte(`

secret: hunter2

`)} if got := d.FromChunk(chunk); got != nil { - t.Errorf("FromChunk() should return nil when disabled, got %q", string(got.Chunk.Data)) + t.Errorf("FromChunk() should return nil when disabled, got %q", string(got.Data)) } } @@ -439,8 +506,8 @@ func TestHTML_FeatureFlagEnabled(t *testing.T) { if got == nil { t.Fatal("FromChunk() returned nil, want decoded chunk") } - if string(got.Chunk.Data) != "secret: hunter2" { - t.Errorf("FromChunk() data = %q, want %q", string(got.Chunk.Data), "secret: hunter2") + if string(got.Data) != "secret: hunter2" { + t.Errorf("FromChunk() data = %q, want %q", string(got.Data), "secret: hunter2") } } @@ -462,6 +529,25 @@ func TestLooksLikeHTML(t *testing.T) { {"XML-like", "content", true}, {"just less-than", "a < b", false}, {"html entity only", "& <", false}, + + // ASPX / namespace-prefixed tags (literal form) + {"aspx namespace tag with space", ``, true}, + {"mso namespace tag with space", ``, true}, + {"multi-segment namespace tag", ``, true}, + + // Entity-encoded HTML tags (positive) + {"entity-encoded div with space", `<div class="foo">`, true}, + {"entity-encoded p with gt terminator", `<p>hello</p>`, true}, + {"entity-encoded self-closing br", `<br/>`, true}, + {"entity-encoded namespace tag", `<asp:Content runat="server">`, true}, + + // Entity-encoded false positives that must NOT match + {"comparison operator entity-encoded", `x < maxRetries exceeded`, false}, + {"comparison operator single word", `result < threshold`, false}, + {"template placeholder entity-encoded", `<YOUR_API_KEY>`, false}, + {"double-encoded entity not matched", `&lt;div>`, false}, + {"json with digit after lt", `{"lt": "<3"}`, false}, + {"sql comparison bare lt", `SELECT * FROM t WHERE a < b`, false}, } for _, tt := range tests { diff --git a/pkg/detectors/abstract/abstract.go b/pkg/detectors/abstract/abstract.go index cd2b63bdfe2c..620571b5217c 100644 --- a/pkg/detectors/abstract/abstract.go +++ b/pkg/detectors/abstract/abstract.go @@ -3,10 +3,11 @@ package abstract import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -80,7 +81,7 @@ func verifyAbstract(ctx context.Context, client *http.Client, resMatch string) ( if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://docs.abstractapi.com/exchange-rates#response-and-error-codes switch res.StatusCode { diff --git a/pkg/detectors/abuseipdb/abuseipdb.go b/pkg/detectors/abuseipdb/abuseipdb.go index 906a614af37f..685f931d8e4d 100644 --- a/pkg/detectors/abuseipdb/abuseipdb.go +++ b/pkg/detectors/abuseipdb/abuseipdb.go @@ -83,7 +83,7 @@ func verifyAbuseIPDB(ctx context.Context, client *http.Client, resMatch string) if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/abyssale/abyssale.go b/pkg/detectors/abyssale/abyssale.go index 6865f811a35c..3ff29716a089 100644 --- a/pkg/detectors/abyssale/abyssale.go +++ b/pkg/detectors/abyssale/abyssale.go @@ -81,7 +81,7 @@ func verifyAbyssale(ctx context.Context, client *http.Client, resMatch string) ( if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/accuweather/v1/accuweather.go b/pkg/detectors/accuweather/v1/accuweather.go index e2f61563d77e..32a167ea3b94 100644 --- a/pkg/detectors/accuweather/v1/accuweather.go +++ b/pkg/detectors/accuweather/v1/accuweather.go @@ -93,7 +93,7 @@ func verifyAccuweather(ctx context.Context, client *http.Client, key string) (bo if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://developer.accuweather.com/accuweather-locations-api/apis/get/locations/v1/cities/autocomplete switch res.StatusCode { diff --git a/pkg/detectors/adafruitio/adafruitio.go b/pkg/detectors/adafruitio/adafruitio.go index 95145920b46c..b7c3555d7c9b 100644 --- a/pkg/detectors/adafruitio/adafruitio.go +++ b/pkg/detectors/adafruitio/adafruitio.go @@ -79,7 +79,7 @@ func verifyAdafruitIO(ctx context.Context, client *http.Client, resMatch string) if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://learn.adafruit.com/adafruit-io/http-status-codes switch res.StatusCode { diff --git a/pkg/detectors/adzuna/adzuna.go b/pkg/detectors/adzuna/adzuna.go index 0cdb23b34009..f662049c650f 100644 --- a/pkg/detectors/adzuna/adzuna.go +++ b/pkg/detectors/adzuna/adzuna.go @@ -91,7 +91,7 @@ func verifyAdzuna(ctx context.Context, client *http.Client, resMatch, resIdMatch if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://developer.adzuna.com/overview switch res.StatusCode { diff --git a/pkg/detectors/aeroworkflow/aeroworkflow.go b/pkg/detectors/aeroworkflow/aeroworkflow.go index 10e1c63015f8..2c798a56d3b5 100644 --- a/pkg/detectors/aeroworkflow/aeroworkflow.go +++ b/pkg/detectors/aeroworkflow/aeroworkflow.go @@ -93,7 +93,7 @@ func verifyAeroworkflow(ctx context.Context, client *http.Client, resMatch, resI if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://api.aeroworkflow.com/swagger/index.html switch res.StatusCode { diff --git a/pkg/detectors/agora/agora.go b/pkg/detectors/agora/agora.go index a2284b4ce91a..328497a0be0c 100644 --- a/pkg/detectors/agora/agora.go +++ b/pkg/detectors/agora/agora.go @@ -99,7 +99,7 @@ func verifyAgora(ctx context.Context, client *http.Client, resMatch, resSecret s if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://docs.agora.io/en/voice-calling/reference/agora-console-rest-api#get-all-projects switch res.StatusCode { diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.go index b17bb82bb3d6..27fc01e46424 100644 --- a/pkg/detectors/aha/aha.go +++ b/pkg/detectors/aha/aha.go @@ -106,7 +106,7 @@ func verifyAha(ctx context.Context, client *http.Client, resMatch, resURLMatch s if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // https://www.aha.io/api switch res.StatusCode { diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go index 5bb5773b20ca..255c05acf76d 100644 --- a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go +++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey.go @@ -56,6 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_AirbrakeProjectKey, Raw: []byte(key), RawV2: []byte(key + id), + SecretParts: map[string]string{"key": key, "id": id}, } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/airbrake/", @@ -70,9 +71,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyAirbrakeProjectKey(ctx, client, key, id) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - if isVerified { - s1.SecretParts = map[string]string{"key": key} - } } results = append(results, s1) diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey.go index 1dc23ba29396..cdce0c88a043 100644 --- a/pkg/detectors/airbrakeuserkey/airbrakeuserkey.go +++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey.go @@ -50,6 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/airbrake/", }, + SecretParts: map[string]string{"key": key}, } if verify { @@ -61,9 +62,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyAirbrakeUserKey(ctx, client, key) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - if isVerified { - s1.SecretParts = map[string]string{"key": key} - } } results = append(results, s1) diff --git a/pkg/detectors/airtableoauth/airtableoauth.go b/pkg/detectors/airtableoauth/airtableoauth.go index d5d15d6a8c44..5d7c6bd619e5 100644 --- a/pkg/detectors/airtableoauth/airtableoauth.go +++ b/pkg/detectors/airtableoauth/airtableoauth.go @@ -47,6 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_AirtableOAuth, Raw: []byte(match), + SecretParts: map[string]string{"token": match}, } if verify { @@ -59,10 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{"token": match} - } } results = append(results, s1) diff --git a/pkg/detectors/airtablepersonalaccesstoken/airtablepersonalaccesstoken.go b/pkg/detectors/airtablepersonalaccesstoken/airtablepersonalaccesstoken.go index ba6820d8559f..3bd31155e157 100644 --- a/pkg/detectors/airtablepersonalaccesstoken/airtablepersonalaccesstoken.go +++ b/pkg/detectors/airtablepersonalaccesstoken/airtablepersonalaccesstoken.go @@ -42,6 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_AirtablePersonalAccessToken, Raw: []byte(match), + SecretParts: map[string]string{"token": match}, } if verify { @@ -54,10 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{"token": match} - } } results = append(results, s1) diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey.go b/pkg/detectors/algoliaadminkey/algoliaadminkey.go index 37f99070651f..a3cdfc818229 100644 --- a/pkg/detectors/algoliaadminkey/algoliaadminkey.go +++ b/pkg/detectors/algoliaadminkey/algoliaadminkey.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "slices" "strings" @@ -68,12 +69,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // Test matches. for key := range keyMatches { for id := range idMatches { - if invalidHosts.Exists(id) { - logger.V(3).Info("Skipping application id: no such host", "host", id) - delete(idMatches, id) - continue - } - r := detectors.Result{ DetectorType: detector_typepb.DetectorType_AlgoliaAdminKey, Raw: []byte(key), @@ -85,17 +80,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - // Verify if the key is a valid Algolia Admin Key. - isVerified, extraData, verificationErr := verifyMatch(ctx, id, key) - r.Verified = isVerified - r.ExtraData = extraData - if verificationErr != nil { - if errors.Is(verificationErr, errNoHost) { - invalidHosts.Set(id, struct{}{}) - continue + if invalidHosts.Exists(id) { + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + logger.V(3).Info("Skipping application id: no such host", "host", id) + r.SetVerificationError(errNoHost, key) + } else { + // Verify if the key is a valid Algolia Admin Key. + isVerified, extraData, verificationErr := verifyMatch(ctx, id, key) + r.Verified = isVerified + r.ExtraData = extraData + if verificationErr != nil { + var dnsErr *net.DNSError + if errors.As(verificationErr, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(id, struct{}{}) + } + + r.SetVerificationError(verificationErr, key) } - - r.SetVerificationError(verificationErr, key) } } @@ -127,11 +131,6 @@ func verifyMatch(ctx context.Context, appId, apiKey string) (bool, map[string]st res, err := client.Do(req) if err != nil { - // lookup xyz.algolia.net: no such host - if strings.Contains(err.Error(), "no such host") { - return false, nil, errNoHost - } - return false, nil, err } defer func() { diff --git a/pkg/detectors/alibaba/alibaba.go b/pkg/detectors/alibaba/alibaba.go index 91417aae6c09..bc8a7bb1663a 100644 --- a/pkg/detectors/alibaba/alibaba.go +++ b/pkg/detectors/alibaba/alibaba.go @@ -75,9 +75,9 @@ func GetSignature(input, key string) string { } func buildStringToSign(method, input string) string { - filter := strings.Replace(input, "+", "%20", -1) - filter = strings.Replace(filter, "%7E", "~", -1) - filter = strings.Replace(filter, "*", "%2A", -1) + filter := strings.ReplaceAll(input, "+", "%20") + filter = strings.ReplaceAll(filter, "%7E", "~") + filter = strings.ReplaceAll(filter, "*", "%2A") filter = method + "&%2F&" + url.QueryEscape(filter) return filter } @@ -155,7 +155,7 @@ func verifyAlibaba(ctx context.Context, client *http.Client, resIdMatch, resMatc if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() var alibabaResp alibabaResp if err = json.NewDecoder(res.Body).Decode(&alibabaResp); err != nil { diff --git a/pkg/detectors/anthropic/anthropic.go b/pkg/detectors/anthropic/anthropic.go index bcd09c32e0ee..166c5d65e7a8 100644 --- a/pkg/detectors/anthropic/anthropic.go +++ b/pkg/detectors/anthropic/anthropic.go @@ -2,8 +2,9 @@ package anthropic import ( "context" - "errors" + "encoding/json" "fmt" + "io" "net/http" "strings" @@ -50,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_Anthropic, Raw: []byte(keyMatch), ExtraData: make(map[string]string), + SecretParts: map[string]string{"key": keyMatch}, } if verify { @@ -58,28 +60,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result client = defaultClient } - isAdminKey := isAdminKey(keyMatch) - var isVerified bool - var err error + var ( + isVerified bool + verificationErr error + ) - if isAdminKey { - isVerified, err = verifyAnthropicKey(ctx, client, adminKeyEndpoint, keyMatch) + if isAdminKey(keyMatch) { s1.ExtraData["Type"] = "Admin Key" - } else if !isAdminKey { - isVerified, err = verifyAnthropicKey(ctx, client, apiKeyEndpoint, keyMatch) - s1.ExtraData["Type"] = "API Key" + isVerified, verificationErr = verifyAnthropicKey(ctx, client, adminKeyEndpoint, keyMatch) } else { - return nil, errors.New("unknown key type detected for anthropic") + s1.ExtraData["Type"] = "API Key" + isVerified, verificationErr = verifyAnthropicKey(ctx, client, apiKeyEndpoint, keyMatch) } s1.Verified = isVerified - s1.SetVerificationError(err, keyMatch) - - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": keyMatch, - } - } + s1.SetVerificationError(verificationErr, keyMatch) } results = append(results, s1) @@ -100,7 +95,7 @@ Endpoints: func verifyAnthropicKey(ctx context.Context, client *http.Client, endpoint, key string) (bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody) if err != nil { - return false, nil + return false, err } req.Header.Set("x-api-key", key) @@ -111,7 +106,10 @@ func verifyAnthropicKey(ctx context.Context, client *http.Client, endpoint, key if err != nil { return false, err } - defer res.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() switch res.StatusCode { case http.StatusOK: @@ -122,8 +120,37 @@ func verifyAnthropicKey(ctx context.Context, client *http.Client, endpoint, key return false, nil default: - return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + // A 400 is invalid_request_error, never a bad-key signal (that's always 401), + // so it must stay indeterminate rather than count as not-live. + return false, apiError(res) + } +} + +// anthropicErrorResponse is the Anthropic API's error envelope for non-2xx responses. +// See https://platform.claude.com/docs/en/api/errors. +type anthropicErrorResponse struct { + Error struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` +} + +const maxErrorBodySize = 4 << 10 // cap how much of the error body we bother parsing + +// apiError enriches an unexpected status with the API's own error type/message when present. +func apiError(res *http.Response) error { + body, err := io.ReadAll(io.LimitReader(res.Body, maxErrorBodySize)) + if err != nil { + return fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } + + var apiErr anthropicErrorResponse + if err := json.Unmarshal(body, &apiErr); err != nil || apiErr.Error.Type == "" { + return fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } + + return fmt.Errorf("unexpected HTTP response status %d (%s: %s)", + res.StatusCode, apiErr.Error.Type, apiErr.Error.Message) } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/anthropic/anthropic_test.go b/pkg/detectors/anthropic/anthropic_test.go index b4ea6730e82a..2883527af89a 100644 --- a/pkg/detectors/anthropic/anthropic_test.go +++ b/pkg/detectors/anthropic/anthropic_test.go @@ -2,15 +2,101 @@ package anthropic import ( "context" + "net/http" "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) +func TestAnthropic_VerifyMatch(t *testing.T) { + const key = "sk-ant-api03-Dtjm9IZ_rYhS_ihHLZmPXhjJ6PN8UPp7vNO7qO3735RRDpf8xbWGinsch0McONXznUm-4KWoA7WU2otvvwHBR5QRjiLakAA" + + tests := []struct { + name string + statusCode int + body string + wantVerified bool + wantErr bool + // wantErrContains, when set, must appear in the verification error. It pins that + // the API's own explanation survives into the error rather than being discarded. + wantErrContains string + }{ + { + name: "200 is verified", + statusCode: http.StatusOK, + body: `{"data":[]}`, + wantVerified: true, + }, + { + name: "401 is determinate not-live", + statusCode: http.StatusUnauthorized, + body: `{"type":"error","error":{"type":"authentication_error","message":"API key is invalid."}}`, + }, + { + name: "404 is determinate not-live", + statusCode: http.StatusNotFound, + body: `{"type":"error","error":{"type":"not_found_error","message":"Not found"}}`, + }, + { + // 400 is invalid_request_error: the API returns it for a malformed request, + // never for the state of the key (bad keys return 401). It must stay + // indeterminate, otherwise a request mangled in transit silently marks a live + // key as not-verified. + name: "400 stays indeterminate and keeps the API message", + statusCode: http.StatusBadRequest, + body: `{"type":"error","error":{"type":"invalid_request_error","message":"anthropic-version: header is required"}}`, + wantErr: true, + wantErrContains: "anthropic-version: header is required", + }, + { + name: "429 stays indeterminate", + statusCode: http.StatusTooManyRequests, + body: `{"type":"error","error":{"type":"rate_limit_error","message":"Number of requests has exceeded your rate limit"}}`, + wantErr: true, + wantErrContains: "rate_limit_error", + }, + { + name: "500 stays indeterminate", + statusCode: http.StatusInternalServerError, + body: `{"type":"error","error":{"type":"api_error","message":"Internal server error"}}`, + wantErr: true, + }, + { + // A gateway that returns a non-JSON body must still degrade to a plain + // status-code error rather than panicking or losing the error entirely. + name: "unparseable body still yields an error", + statusCode: http.StatusBadRequest, + body: `502 Bad Gateway`, + wantErr: true, + wantErrContains: "unexpected HTTP response status 400", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := common.ConstantResponseHttpClient(test.statusCode, test.body) + + verified, err := verifyAnthropicKey(context.Background(), client, apiKeyEndpoint, key) + + if test.wantErr { + require.Error(t, err) + if test.wantErrContains != "" { + assert.Contains(t, err.Error(), test.wantErrContains) + } + } else { + require.NoError(t, err) + } + assert.Equal(t, test.wantVerified, verified) + }) + } +} + func TestAnthropic_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) diff --git a/pkg/detectors/anypointoauth2/anypointoauth2.go b/pkg/detectors/anypointoauth2/anypointoauth2.go index 597ced839584..0a951c7c55f8 100644 --- a/pkg/detectors/anypointoauth2/anypointoauth2.go +++ b/pkg/detectors/anypointoauth2/anypointoauth2.go @@ -70,6 +70,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_AnypointOAuth2, Raw: []byte(secret), RawV2: []byte(fmt.Sprintf("%s:%s", id, secret)), + SecretParts: map[string]string{ + "client_id": id, + "client_secret": secret, + }, } if verify { @@ -77,12 +81,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyMatch(ctx, client, id, secret) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - if isVerified { - s1.SecretParts = map[string]string{ - "client_id": id, - "client_secret": secret, - } - } } results = append(results, s1) diff --git a/pkg/detectors/apacta/apacta_test.go b/pkg/detectors/apacta/apacta_test.go index b03975ac3e0d..35ef67642c90 100644 --- a/pkg/detectors/apacta/apacta_test.go +++ b/pkg/detectors/apacta/apacta_test.go @@ -37,7 +37,7 @@ func TestApacta_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"}, @@ -73,7 +73,7 @@ func TestApacta_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/appoptics/appoptics.go b/pkg/detectors/appoptics/appoptics.go deleted file mode 100644 index fdd1805e0d67..000000000000 --- a/pkg/detectors/appoptics/appoptics.go +++ /dev/null @@ -1,94 +0,0 @@ -package appoptics - -import ( - "context" - b64 "encoding/base64" - "fmt" - regexp "github.com/wasilibs/go-re2" - "net/http" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" -) - -type Scanner struct { - client *http.Client -} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appoptics"}) + `\b([0-9a-zA-Z_-]{71})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"appoptics"} -} - -// FromData will find and optionally verify Appoptics secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_AppOptics, - Raw: []byte(resMatch), - SecretParts: map[string]string{"key": resMatch}, - } - - if verify { - client := s.client - if client == nil { - client = defaultClient - } - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.appoptics.com/v1/metrics", nil) - if err != nil { - continue - } - - data := fmt.Sprintf("%s:", resMatch) - sEnc := b64.StdEncoding.EncodeToString([]byte(data)) - - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else if res.StatusCode == 401 { - // The secret is determinately not verified (nothing to do) - } else { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resMatch) - } - } else { - s1.SetVerificationError(err, resMatch) - } - } - - results = append(results, s1) - } - - return results, nil -} - -func (s Scanner) Type() detector_typepb.DetectorType { - return detector_typepb.DetectorType_AppOptics -} - -func (s Scanner) Description() string { - return "AppOptics is a cloud-based infrastructure monitoring service. AppOptics API keys can be used to access and manage monitoring data and configurations." -} diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index 99e4707e2bfa..9881c3464b14 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "strings" @@ -77,33 +78,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for token := range uniqueTokens { for url := range uniqueUrls { - if invalidHosts.Exists(url) { - delete(uniqueUrls, url) - continue - } - s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_ArtifactoryAccessToken, Raw: []byte(token), RawV2: []byte(token + url), + SecretParts: map[string]string{ + "domain": url, + "token": token, + }, } if verify { - isVerified, verificationErr := verifyArtifactory(ctx, s.getClient(), url, token) - s1.Verified = isVerified - if verificationErr != nil { - if errors.Is(verificationErr, errNoHost) { - invalidHosts.Set(url, struct{}{}) - continue - } - - s1.SetVerificationError(verificationErr, token) - - if isVerified { - s1.SecretParts = map[string]string{ - "domain": url, - "token": token, + if invalidHosts.Exists(url) { + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + s1.SetVerificationError(errNoHost, token) + } else { + isVerified, verificationErr := verifyArtifactory(ctx, s.getClient(), url, token) + s1.Verified = isVerified + if verificationErr != nil { + var dnsErr *net.DNSError + if errors.As(verificationErr, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(url, struct{}{}) } + + s1.SetVerificationError(verificationErr, token) } } } @@ -126,11 +127,6 @@ func verifyArtifactory(ctx context.Context, client *http.Client, resURLMatch, re resp, err := client.Do(req) if err != nil { - // lookup foo.jfrog.io: no such host - if strings.Contains(err.Error(), "no such host") { - return false, errNoHost - } - return false, err } diff --git a/pkg/detectors/artifactoryreferencetoken/artifactoryreferencetoken.go b/pkg/detectors/artifactoryreferencetoken/artifactoryreferencetoken.go index 6f45cd8b80f8..09e8599f7c88 100644 --- a/pkg/detectors/artifactoryreferencetoken/artifactoryreferencetoken.go +++ b/pkg/detectors/artifactoryreferencetoken/artifactoryreferencetoken.go @@ -84,6 +84,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_ArtifactoryReferenceToken, Raw: []byte(token), RawV2: []byte(token + url), + SecretParts: map[string]string{ + "domain": url, + "token": token, + }, } if verify { @@ -97,13 +101,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.SetVerificationError(verificationErr, token) } - - if isVerified { - s1.SecretParts = map[string]string{ - "domain": url, - "token": token, - } - } } results = append(results, s1) diff --git a/pkg/detectors/asanaoauth/asanaoauth.go b/pkg/detectors/asanaoauth/asanaoauth.go index 5f5cad550be1..fd16c3ab36ca 100644 --- a/pkg/detectors/asanaoauth/asanaoauth.go +++ b/pkg/detectors/asanaoauth/asanaoauth.go @@ -44,13 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_AsanaOauth, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { isVerified, err := verifyMatch(ctx, client, resMatch) s1.Verified = isVerified s1.SetVerificationError(err, resMatch) - s1.SecretParts = map[string]string{"key": resMatch} } results = append(results, s1) diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken.go index c077004321d6..41b8b7a5098a 100644 --- a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken.go +++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken.go @@ -46,15 +46,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_AsanaPersonalAccessToken, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { isVerified, err := verifyMatch(ctx, client, resMatch) s1.Verified = isVerified s1.SetVerificationError(err, resMatch) - if isVerified { - s1.SecretParts = map[string]string{"key": resMatch} - } } results = append(results, s1) diff --git a/pkg/detectors/atlassian/v1/atlassian.go b/pkg/detectors/atlassian/v1/atlassian.go index a0ce2cdbd0e9..689c33b8a826 100644 --- a/pkg/detectors/atlassian/v1/atlassian.go +++ b/pkg/detectors/atlassian/v1/atlassian.go @@ -35,7 +35,7 @@ var _ detectors.Versioner = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian"}) + `\b([a-zA-Z-0-9]{24})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian"}) + `\b([a-zA-Z0-9]{24})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -66,6 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: map[string]string{"key": match}, } if verify { @@ -80,12 +81,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["Organization"] = orgResponse.Data[0].Attributes.Name } s1.SetVerificationError(verificationErr, match) - - if isVerified { - s1.SecretParts = map[string]string{ - "key": match, - } - } } results = append(results, s1) diff --git a/pkg/detectors/atlassian/v1/atlassian_test.go b/pkg/detectors/atlassian/v1/atlassian_test.go index 335f4bdaab8c..3724bde4122c 100644 --- a/pkg/detectors/atlassian/v1/atlassian_test.go +++ b/pkg/detectors/atlassian/v1/atlassian_test.go @@ -41,6 +41,15 @@ func TestAtlassian_Pattern(t *testing.T) { `, want: []string{"r6RkiQao3PgqY9MOKtonpJdU"}, }, + { + name: "confluence macro-id near atlassian url - no match", + input: ` + link + + + `, + want: nil, + }, } for _, test := range tests { diff --git a/pkg/detectors/atlassian/v2/atlassian.go b/pkg/detectors/atlassian/v2/atlassian.go index 6e478707bfca..cbbbc947f619 100644 --- a/pkg/detectors/atlassian/v2/atlassian.go +++ b/pkg/detectors/atlassian/v2/atlassian.go @@ -75,6 +75,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for match := range uniqueMatches { for orgId := range uniqueOrgIdMatches { + secretParts := map[string]string{"key": match} + if orgId != "" { + secretParts["organization_id"] = orgId + } s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Atlassian, Raw: []byte(match), @@ -82,6 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: secretParts, } if verify { @@ -96,14 +101,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["Organization"] = orgResponse.Data[0].Attributes.Name } s1.SetVerificationError(verificationErr, match) - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": match, - } - if orgId != "" { - s1.SecretParts["organization_id"] = orgId - } - } } results = append(results, s1) diff --git a/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter.go b/pkg/detectors/atlassiandatacenter/bitbucketdatacenter/bitbucketdatacenter.go similarity index 70% rename from pkg/detectors/bitbucketdatacenter/bitbucketdatacenter.go rename to pkg/detectors/atlassiandatacenter/bitbucketdatacenter/bitbucketdatacenter.go index 30839b2cec9d..1a9dba07222b 100644 --- a/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter.go +++ b/pkg/detectors/atlassiandatacenter/bitbucketdatacenter/bitbucketdatacenter.go @@ -3,13 +3,13 @@ package bitbucketdatacenter import ( "context" "fmt" - "io" "net/http" "strings" regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -30,8 +30,7 @@ var ( // and are usually between the length of 40-50 character // consisting of both alphanumeric and some special character like +, _, @ and etc userPat = regexp.MustCompile(`\b(BBDC-[A-Za-z0-9+/@_-]{40,50})(?:[^A-Za-z0-9+/@_-]|$)`) - - urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "bitbucket"}) + `(https?://[a-zA-Z0-9.-]+(?::\d+)?)`) + urlPat = atlassiandatacenter.GetURLPat([]string{"atlassian", "bitbucket"}) ) func (s Scanner) Keywords() []string { @@ -52,24 +51,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } - foundURLs := make(map[string]struct{}) - for _, match := range urlPat.FindAllStringSubmatch(dataStr, -1) { - foundURLs[match[1]] = struct{}{} - } - - endpoints := make([]string, 0, len(foundURLs)) - for endpoint := range foundURLs { - endpoints = append(endpoints, endpoint) - } - - var uniqueUrls = make(map[string]struct{}) - for _, endpoint := range s.Endpoints(endpoints...) { - uniqueUrls[endpoint] = struct{}{} - } + endpoints := atlassiandatacenter.FindEndpoints(dataStr, urlPat, s.Endpoints) // create combination results that can be verified for secret := range uniqueSecretPat { - for bitBucketURL := range uniqueUrls { + for _, bitBucketURL := range endpoints { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_BitbucketDataCenter, Raw: []byte(secret), @@ -109,30 +95,8 @@ func verifyMatch(ctx context.Context, client *http.Client, secretPat, baseURL st q.Set("limit", "1") u.RawQuery = q.Encode() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) - if err != nil { - return false, err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", secretPat)) - resp, err := client.Do(req) - if err != nil { - return false, err - } - - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - - switch resp.StatusCode { - case http.StatusOK: - return true, nil - case http.StatusUnauthorized: - return false, nil - default: - return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode) - } + isVerified, _, err := atlassiandatacenter.MakeVerifyRequest(ctx, client, u.String(), secretPat) + return isVerified, err } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter_test.go b/pkg/detectors/atlassiandatacenter/bitbucketdatacenter/bitbucketdatacenter_test.go similarity index 100% rename from pkg/detectors/bitbucketdatacenter/bitbucketdatacenter_test.go rename to pkg/detectors/atlassiandatacenter/bitbucketdatacenter/bitbucketdatacenter_test.go diff --git a/pkg/detectors/atlassiandatacenter/common.go b/pkg/detectors/atlassiandatacenter/common.go new file mode 100644 index 000000000000..e5ac437dfaa6 --- /dev/null +++ b/pkg/detectors/atlassiandatacenter/common.go @@ -0,0 +1,124 @@ +package atlassiandatacenter + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" +) + +// GetDCTokenPat returns a compiled regex that matches Atlassian Data Center PATs +// (Jira DC and Confluence DC style) scoped to the given keyword prefixes. +// +// PATs are 44-char base64 strings decoding to ":". +// The first character is always M, N, or O because the numeric ID begins with +// an ASCII digit (0x30–0x39). The trailing boundary prevents matching substrings +// of longer base64 strings or base64-padded tokens. +// +// This does not apply to Bitbucket DC tokens, which use a BBDC- prefix format. +func GetDCTokenPat(prefixes []string) *regexp.Regexp { + return regexp.MustCompile( + detectors.PrefixRegex(prefixes) + `\b([MNO][A-Za-z0-9+/]{43})(?:[^A-Za-z0-9+/=]|\z)`, + ) +} + +// GetURLPat returns a compiled regex that matches self-hosted Atlassian instance +// URLs (scheme + alphanumeric-starting host + optional port up to 5 digits), +// scoped to the given keyword prefixes. Callers should store the result in a +// package-level var so the regex is compiled once at init time rather than per chunk. +func GetURLPat(prefixes []string) *regexp.Regexp { + return regexp.MustCompile(detectors.PrefixRegex(prefixes) + `(https?://[a-zA-Z0-9][a-zA-Z0-9.\-]*(?::\d{1,5})?)`) +} + +// FindEndpoints extracts all URLs matching urlPat from data, passes them through +// the resolve function (typically s.Endpoints), deduplicates the results, and +// returns them as a slice with trailing slashes stripped. +func FindEndpoints(data string, urlPat *regexp.Regexp, resolve func(...string) []string) []string { + seen := make(map[string]struct{}) + for _, m := range urlPat.FindAllStringSubmatch(data, -1) { + seen[m[1]] = struct{}{} + } + + raw := make([]string, 0, len(seen)) + for u := range seen { + raw = append(raw, u) + } + + resolved := make(map[string]struct{}) + for _, u := range resolve(raw...) { + resolved[strings.TrimRight(u, "/")] = struct{}{} + } + + result := make([]string, 0, len(resolved)) + for u := range resolved { + result = append(result, u) + } + return result +} + +// IsStructuralPAT decodes a candidate base64 string and checks that it matches +// the ":" structure used by Jira and Confluence DC PATs: +// one or more ASCII digits, a colon, then at least one more byte. +func IsStructuralPAT(candidate string) bool { + raw, err := base64.StdEncoding.DecodeString(candidate) + if err != nil { + return false + } + colon := bytes.IndexByte(raw, ':') + if colon <= 0 || colon == len(raw)-1 { + return false + } + for _, b := range raw[:colon] { + if b < '0' || b > '9' { + return false + } + } + return true +} + +// MakeVerifyRequest sends a Bearer-authenticated GET request to fullURL and +// interprets the response: +// - 200: returns (true, decoded JSON body as map or nil if unparseable, nil) +// - 401: returns (false, nil, nil) +// - other: returns (false, nil, error describing the unexpected status) +// +// A non-nil error is also returned for network failures. +// Callers that need fields from the response body (e.g. display name, email) +// can read them from the returned map; callers that don't need the body can +// ignore it. +func MakeVerifyRequest(ctx context.Context, client *http.Client, fullURL, token string) (bool, map[string]any, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, http.NoBody) + if err != nil { + return false, nil, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + var body map[string]any + _ = json.NewDecoder(resp.Body).Decode(&body) + return true, body, nil + case http.StatusUnauthorized: + return false, nil, nil + default: + return false, nil, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode) + } +} diff --git a/pkg/detectors/atlassiandatacenter/common_test.go b/pkg/detectors/atlassiandatacenter/common_test.go new file mode 100644 index 000000000000..9915addd8cfc --- /dev/null +++ b/pkg/detectors/atlassiandatacenter/common_test.go @@ -0,0 +1,305 @@ +package atlassiandatacenter + +import ( + "context" + "encoding/base64" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/h2non/gock.v1" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" +) + +// Real-format sample PATs that decode to ":". +const ( + // Jira DC sample. + jiraToken = "NTg4OTI1Mzk1OTA1OiBb9S4WPEoK6cmOe6pq6VO0lt6M" + + // Confluence DC samples. + confluenceToken1 = "NTk3MjQzOTIyNTAwOtFOuTsHRIp1E81GApKpC2xpEzfz" + confluenceToken2 = "NDc4MjM3OTUxMzk2OopoSkTDTnBcWIw0Wa4bico9zOLK" + + // 44-char base64 that starts with [MNO] (passes the regex) but decodes + // to bytes with no colon — must be rejected by IsStructuralPAT. + nonStructural = "MAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +) + +func encode(b []byte) string { return base64.StdEncoding.EncodeToString(b) } + +func TestIsStructuralPAT(t *testing.T) { + tests := []struct { + name string + candidate string + want bool + }{ + { + name: "valid real Jira token", + candidate: jiraToken, + want: true, + }, + { + name: "valid real Confluence token 1", + candidate: confluenceToken1, + want: true, + }, + { + name: "valid real Confluence token 2", + candidate: confluenceToken2, + want: true, + }, + { + name: "valid - digits before colon", + candidate: encode([]byte("123456789012:01234567890123456789")), + want: true, + }, + { + name: "invalid base64", + candidate: "!!!not-base64!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", + want: false, + }, + { + name: "no colon in decoded bytes", + candidate: encode([]byte("588925395905012345678901234567890")), + want: false, + }, + { + name: "colon at position 0", + candidate: encode([]byte(":01234567890123456789012345678901")), + want: false, + }, + { + name: "colon at last position", + candidate: encode([]byte("58892539590501234567890123456789:")), + want: false, + }, + { + // decodes to "0a:xxx..." — 'a' is not a digit + name: "non-digit before colon", + candidate: "MGE6eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4", + want: false, + }, + { + name: "non-structural: passes regex but no colon in decoded bytes", + candidate: nonStructural, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsStructuralPAT(tt.candidate)) + }) + } +} + +func TestGetDCTokenPat(t *testing.T) { + pat := GetDCTokenPat([]string{"jira", "atlassian"}) + + // Known-valid token — must match and capture exactly the token (no trailing char). + m := pat.FindStringSubmatch("jira token: " + jiraToken) + require.NotNil(t, m, "expected a match") + assert.Equal(t, jiraToken, m[1], "captured group should be exactly the token") + + // Token followed by newline — must match and NOT capture the newline. + m2 := pat.FindStringSubmatch("jira token: " + jiraToken + "\n") + require.NotNil(t, m2) + assert.Equal(t, jiraToken, m2[1], "captured group must not include the trailing newline") + + // Token starts with 'A' — not M/N/O — must not match. + assert.Nil(t, pat.FindStringSubmatch("jira token: ATg4OTI1Mzk1OTA1OiBb9S4WPEoK6cmOe6pq6VO0lt6M")) + + // Token followed by base64 padding — trailing boundary must reject it. + assert.Nil(t, pat.FindStringSubmatch("jira token: "+jiraToken+"=")) + + // Token followed by more base64 chars — must not match (longer string). + assert.Nil(t, pat.FindStringSubmatch("jira token: "+jiraToken+"AAAA")) +} + +func TestFindEndpoints(t *testing.T) { + urlPat := GetURLPat([]string{"jira", "atlassian"}) + + // identity resolver: returns exactly what it receives (simulates UseFoundEndpoints only) + identity := func(urls ...string) []string { return urls } + + tests := []struct { + name string + data string + resolve func(...string) []string + want []string + }{ + { + name: "URL near keyword is returned", + data: "jira url: https://jira.corp.com", + resolve: identity, + want: []string{"https://jira.corp.com"}, + }, + { + name: "URL not near any keyword is ignored", + data: "unrelated url: https://example.com", + resolve: identity, + want: []string{}, + }, + { + name: "duplicate URLs in data are deduplicated", + data: "jira: https://jira.corp.com\natlassian: https://jira.corp.com", + resolve: identity, + want: []string{"https://jira.corp.com"}, + }, + { + name: "trailing slash is stripped", + data: "jira url: https://jira.corp.com/", + resolve: identity, + want: []string{"https://jira.corp.com"}, + }, + { + name: "URL with port is accepted", + data: "jira url: https://jira.corp.com:8443", + resolve: identity, + want: []string{"https://jira.corp.com:8443"}, + }, + { + name: "multiple distinct URLs are all returned", + data: "jira prod: https://jira.prod.com\natlassian staging: https://jira.staging.com", + resolve: identity, + want: []string{"https://jira.prod.com", "https://jira.staging.com"}, + }, + { + name: "resolve can inject configured endpoints not in data", + data: "no urls here but jira keyword present", + resolve: func(urls ...string) []string { + return append(urls, "https://configured.jira.com") + }, + want: []string{"https://configured.jira.com"}, + }, + { + name: "resolve can filter out URLs", + data: "jira url: https://jira.corp.com", + resolve: func(urls ...string) []string { + return []string{} // simulate UseFoundEndpoints(false) with no configured endpoint + }, + want: []string{}, + }, + { + name: "no data returns empty slice", + data: "", + resolve: identity, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FindEndpoints(tt.data, urlPat, tt.resolve) + assert.ElementsMatch(t, tt.want, got) + }) + } +} + +func TestMakeVerifyRequest(t *testing.T) { + const testURL = "http://dc.example.com/rest/api/test" + const testToken = "NTg4OTI1Mzk1OTA1OiBb9S4WPEoK6cmOe6pq6VO0lt6M" + + t.Run("200: verified=true, body decoded", func(t *testing.T) { + client := common.SaneHttpClient() + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + gock.New("http://dc.example.com"). + Get("/rest/api/test"). + MatchHeader("Authorization", "Bearer "+testToken). + MatchHeader("Accept", "application/json"). + Reply(http.StatusOK). + JSON(map[string]any{"displayName": "Alice", "emailAddress": "alice@example.com"}) + + verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken) + require.NoError(t, err) + assert.True(t, verified) + require.NotNil(t, body) + assert.Equal(t, "Alice", body["displayName"]) + assert.Equal(t, "alice@example.com", body["emailAddress"]) + }) + + t.Run("200: verified=true, body nil when response is not JSON", func(t *testing.T) { + client := common.SaneHttpClient() + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + gock.New("http://dc.example.com"). + Get("/rest/api/test"). + Reply(http.StatusOK). + BodyString("not json") + + verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken) + require.NoError(t, err) + assert.True(t, verified) + assert.Nil(t, body) + }) + + t.Run("401: verified=false, no error", func(t *testing.T) { + client := common.SaneHttpClient() + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + gock.New("http://dc.example.com"). + Get("/rest/api/test"). + Reply(http.StatusUnauthorized) + + verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken) + require.NoError(t, err) + assert.False(t, verified) + assert.Nil(t, body) + }) + + t.Run("unexpected status: verified=false, error returned", func(t *testing.T) { + client := common.SaneHttpClient() + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + gock.New("http://dc.example.com"). + Get("/rest/api/test"). + Reply(http.StatusInternalServerError) + + verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken) + require.Error(t, err) + assert.False(t, verified) + assert.Nil(t, body) + }) + + t.Run("uses GET method", func(t *testing.T) { + client := common.SaneHttpClient() + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + // gock.Get only matches GET; a POST would not match and would error. + gock.New("http://dc.example.com"). + Get("/rest/api/test"). + Reply(http.StatusOK) + + _, _, err := MakeVerifyRequest(context.Background(), client, testURL, testToken) + require.NoError(t, err) + assert.True(t, gock.IsDone(), "gock interceptor was not matched — request may not have been GET") + }) + + t.Run("propagates network error", func(t *testing.T) { + client := common.SaneHttpClient() + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately to force a network error + + verified, body, err := MakeVerifyRequest(ctx, client, testURL, testToken) + assert.Error(t, err) + assert.False(t, verified) + assert.Nil(t, body) + }) +} diff --git a/pkg/detectors/confluencedatacenter/confluencedatacenter.go b/pkg/detectors/atlassiandatacenter/confluencedatacenter/confluencedatacenter.go similarity index 62% rename from pkg/detectors/confluencedatacenter/confluencedatacenter.go rename to pkg/detectors/atlassiandatacenter/confluencedatacenter/confluencedatacenter.go index 3eb6aae6b574..492ffffb9bb7 100644 --- a/pkg/detectors/confluencedatacenter/confluencedatacenter.go +++ b/pkg/detectors/atlassiandatacenter/confluencedatacenter/confluencedatacenter.go @@ -1,19 +1,15 @@ package confluencedatacenter import ( - "bytes" "context" - "encoding/base64" "errors" "fmt" - "io" "net/http" "strings" - regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -35,12 +31,9 @@ var ( keywords = []string{"confluence", "atlassian", "wiki"} - // 44-char base64 PAT; decoded form must match the structural check below. - tokenPat = regexp.MustCompile(detectors.PrefixRegex(keywords) + `\b([MNO][A-Za-z0-9+/]{43})(?:[^A-Za-z0-9+/=]|\z)`) - - // Self-hosted instance URL: scheme + host + optional port. Keyword-scoped - // so unrelated URLs in the same chunk don't get paired with tokens. - urlPat = regexp.MustCompile(detectors.PrefixRegex(keywords) + `\b(https?://[a-zA-Z0-9.\-]+(?::\d+)?)\b`) + // 44-char base64 PAT; decoded form must match the structural check in atlassiandatacenter.IsStructuralPAT. + tokenPat = atlassiandatacenter.GetDCTokenPat(keywords) + urlPat = atlassiandatacenter.GetURLPat(keywords) invalidHosts = simple.NewCache[struct{}]() errNoHost = errors.New("no such host") @@ -65,26 +58,6 @@ func (s Scanner) getClient() *http.Client { return defaultClient } -// isStructuralPAT decodes a candidate base64 string and checks that it matches -// the ":" structure used by Confluence DC PATs: -// one or more ASCII digits, a colon, then at least one more byte. -func isStructuralPAT(candidate string) bool { - raw, err := base64.StdEncoding.DecodeString(candidate) - if err != nil { - return false - } - colon := bytes.IndexByte(raw, ':') - if colon <= 0 || colon == len(raw)-1 { - return false - } - for _, b := range raw[:colon] { - if b < '0' || b > '9' { - return false - } - } - return true -} - func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -93,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if _, seen := uniqueTokens[m[1]]; seen { continue } - if isStructuralPAT(m[1]) { + if atlassiandatacenter.IsStructuralPAT(m[1]) { uniqueTokens[m[1]] = struct{}{} } } @@ -101,20 +74,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return nil, nil } - foundURLs := make([]string, 0) - for _, m := range urlPat.FindAllStringSubmatch(dataStr, -1) { - foundURLs = append(foundURLs, m[1]) - } - uniqueURLs := make(map[string]struct{}) - for _, endpoint := range s.Endpoints(foundURLs...) { - uniqueURLs[strings.TrimRight(endpoint, "/")] = struct{}{} - } + allURLs := atlassiandatacenter.FindEndpoints(dataStr, urlPat, s.Endpoints) // Filter hosts cached as unreachable from prior calls once up front. // invalidHosts may also grow during this call (see the verify branch // below); those are skipped lazily inside the inner loop. - liveURLs := make([]string, 0, len(uniqueURLs)) - for u := range uniqueURLs { + liveURLs := make([]string, 0, len(allURLs)) + for _, u := range allURLs { if !invalidHosts.Exists(u) { liveURLs = append(liveURLs, u) } @@ -175,35 +141,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result func verifyPAT(ctx context.Context, client *http.Client, baseURL, token string) (bool, error) { endpoint := strings.TrimRight(baseURL, "/") + "/rest/api/user/current" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody) - if err != nil { - return false, err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Authorization", "Bearer "+token) - resp, err := client.Do(req) + isVerified, _, err := atlassiandatacenter.MakeVerifyRequest(ctx, client, endpoint, token) if err != nil { if strings.Contains(err.Error(), "no such host") { return false, errNoHost } return false, err } - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - - switch resp.StatusCode { - case http.StatusOK: - return true, nil - case http.StatusUnauthorized: - // Auth header outright rejected — unambiguously an invalid credential. - return false, nil - default: - // 403 included: /rest/api/user/current should always be readable by a - // valid PAT, so a Forbidden here signals something unexpected rather - // than a definitively invalid token. - return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode) - } + return isVerified, nil } diff --git a/pkg/detectors/confluencedatacenter/confluencedatacenter_test.go b/pkg/detectors/atlassiandatacenter/confluencedatacenter/confluencedatacenter_test.go similarity index 100% rename from pkg/detectors/confluencedatacenter/confluencedatacenter_test.go rename to pkg/detectors/atlassiandatacenter/confluencedatacenter/confluencedatacenter_test.go diff --git a/pkg/detectors/jiradatacenterpat/jiradatacenterpat.go b/pkg/detectors/atlassiandatacenter/jiradatacenterpat/jiradatacenterpat.go similarity index 58% rename from pkg/detectors/jiradatacenterpat/jiradatacenterpat.go rename to pkg/detectors/atlassiandatacenter/jiradatacenterpat/jiradatacenterpat.go index 3ed359652369..9a7af9e97502 100644 --- a/pkg/detectors/jiradatacenterpat/jiradatacenterpat.go +++ b/pkg/detectors/atlassiandatacenter/jiradatacenterpat/jiradatacenterpat.go @@ -1,17 +1,11 @@ package jiradatacenterpat import ( - "bytes" "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" "net/http" - regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -31,12 +25,14 @@ var ( var ( defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses + keywords = []string{"jira", "atlassian"} + // PATs are base64-encoded strings of the form <12-digit-id>:<20-random-bytes> (33 bytes, 44 chars, no padding). // Since the first byte is always an ASCII digit (0x30–0x39), the first base64 character is always M, N, or O. // This is also verified by generating 25+ tokens. // The trailing boundary (?:[^A-Za-z0-9+/=]|\z) is used instead of \b to correctly handle tokens ending in + or /. - patPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira", "atlassian"}) + `\b([MNO][A-Za-z0-9+/]{43})(?:[^A-Za-z0-9+/=]|\z)`) - urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira", "atlassian"}) + `(https?://[A-Za-z0-9][A-Za-z0-9.\-]*(?::\d{1,5})?)`) + patPat = atlassiandatacenter.GetDCTokenPat(keywords) + urlPat = atlassiandatacenter.GetURLPat(keywords) ) func (s Scanner) getClient() *http.Client { @@ -48,7 +44,7 @@ func (s Scanner) getClient() *http.Client { // Keywords are used for efficiently pre-filtering chunks. func (s Scanner) Keywords() []string { - return []string{"jira", "atlassian"} + return keywords } // FromData will find and optionally verify Jira Data Center PAT secrets in a given set of bytes. @@ -57,23 +53,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result tokens := make(map[string]struct{}) for _, match := range patPat.FindAllStringSubmatch(dataStr, -1) { - if isStructuralPAT(match[1]) { + if atlassiandatacenter.IsStructuralPAT(match[1]) { tokens[match[1]] = struct{}{} } } - uniqueURLs := make(map[string]struct{}) - for _, match := range urlPat.FindAllStringSubmatch(dataStr, -1) { - uniqueURLs[match[1]] = struct{}{} - } - foundURLs := make([]string, 0, len(uniqueURLs)) - for url := range uniqueURLs { - foundURLs = append(foundURLs, url) - } - endpoints := make(map[string]struct{}) - for _, endpoint := range s.Endpoints(foundURLs...) { - endpoints[endpoint] = struct{}{} - } + endpoints := atlassiandatacenter.FindEndpoints(dataStr, urlPat, s.Endpoints) for token := range tokens { if len(endpoints) == 0 { @@ -87,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } - for endpoint := range endpoints { + for _, endpoint := range endpoints { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_JiraDataCenterPAT, Raw: []byte(token), @@ -127,65 +112,19 @@ func verifyPAT(ctx context.Context, client *http.Client, baseURL, token string) } u.Path = "/rest/api/2/myself" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) - if err != nil { - return false, nil, err + isVerified, body, err := atlassiandatacenter.MakeVerifyRequest(ctx, client, u.String(), token) + if err != nil || !isVerified { + return isVerified, nil, err } - req.Header.Set("Accept", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - - resp, err := client.Do(req) - if err != nil { - return false, nil, err + extraData := map[string]string{"endpoint": baseURL} + if name, ok := body["displayName"].(string); ok { + extraData["display_name"] = name } - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - - switch resp.StatusCode { - case http.StatusOK: - var result map[string]any - extraData := map[string]string{ - "endpoint": baseURL, - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - // 200 confirms the token is valid; failing to decode only means we can't extract extra data. - return true, extraData, nil - } - if name, ok := result["displayName"].(string); ok { - extraData["display_name"] = name - } - if email, ok := result["emailAddress"].(string); ok { - extraData["email_address"] = email - } - return true, extraData, nil - case http.StatusUnauthorized: - return false, nil, nil - default: - return false, nil, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode) - } -} - -// isStructuralPAT decodes a candidate base64 string and checks that it matches -// the ":" structure used by Jira DC PATs: -// one or more ASCII digits, a colon, then at least one more byte. -func isStructuralPAT(candidate string) bool { - raw, err := base64.StdEncoding.DecodeString(candidate) - if err != nil { - return false - } - colon := bytes.IndexByte(raw, ':') - if colon <= 0 || colon == len(raw)-1 { - return false - } - for _, b := range raw[:colon] { - if b < '0' || b > '9' { - return false - } + if email, ok := body["emailAddress"].(string); ok { + extraData["email_address"] = email } - return true + return true, extraData, nil } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/jiradatacenterpat/jiradatacenterpat_test.go b/pkg/detectors/atlassiandatacenter/jiradatacenterpat/jiradatacenterpat_test.go similarity index 84% rename from pkg/detectors/jiradatacenterpat/jiradatacenterpat_test.go rename to pkg/detectors/atlassiandatacenter/jiradatacenterpat/jiradatacenterpat_test.go index 1b78657e4b80..5e8d3e791d5c 100644 --- a/pkg/detectors/jiradatacenterpat/jiradatacenterpat_test.go +++ b/pkg/detectors/atlassiandatacenter/jiradatacenterpat/jiradatacenterpat_test.go @@ -2,7 +2,6 @@ package jiradatacenterpat import ( "context" - "encoding/base64" "fmt" "net/http" "testing" @@ -275,64 +274,6 @@ func TestJiraDataCenterPAT_FromData(t *testing.T) { } } -func TestIsStructuralPAT(t *testing.T) { - encode := func(b []byte) string { return base64.StdEncoding.EncodeToString(b) } - - // helper to build a 33-byte payload with a numeric id and random suffix - numericIDPayload := func(id, suffix string) []byte { - return []byte(id + ":" + suffix) - } - - tests := []struct { - name string - candidate string - want bool - }{ - { - name: "valid real token", - candidate: testToken, - want: true, - }, - { - name: "valid - digits before colon", - candidate: encode(numericIDPayload("123456789012", "01234567890123456789")), - want: true, - }, - { - name: "invalid base64", - candidate: "!!!not-base64!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - want: false, - }, - { - name: "no colon", - candidate: encode([]byte("588925395905012345678901234567890")), - want: false, - }, - { - name: "colon at position 0", - candidate: encode([]byte(":01234567890123456789012345678901")), - want: false, - }, - { - name: "colon at last position", - candidate: encode([]byte("58892539590501234567890123456789:")), - want: false, - }, - { - name: "non-digit before colon", - // decodes to "0a:xxx..." — 'a' is not a digit - candidate: "MGE6eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4", - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, isStructuralPAT(tt.candidate)) - }) - } -} - func TestJiraDataCenterPAT_NoURL(t *testing.T) { d := Scanner{client: common.SaneHttpClient()} diff --git a/pkg/detectors/audd/audd.go b/pkg/detectors/audd/audd.go index 6cf65ed74e68..313191a69205 100644 --- a/pkg/detectors/audd/audd.go +++ b/pkg/detectors/audd/audd.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"status":"success"`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/auth0oauth/auth0oauth.go b/pkg/detectors/auth0oauth/auth0oauth.go index fa2c1f5c5ed6..619cc3f96612 100644 --- a/pkg/detectors/auth0oauth/auth0oauth.go +++ b/pkg/detectors/auth0oauth/auth0oauth.go @@ -27,7 +27,11 @@ var ( clientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b([a-zA-Z0-9_-]{32,60})\b`) clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`) - domainPat = regexp.MustCompile(`\b([a-zA-Z0-9][a-zA-Z0-9._-]*auth0\.com)\b`) // could be part of url + // could be part of url; auth0app.com is Auth0's managed domain for Private Cloud tenants. + // TODO: a bare domain with no leading character (e.g. "auth0.com" or "auth0app.com" with + // no subdomain/prefix at all) never matches, since the leading character class must be + // consumed before the literal "auth0" substring. Pre-existing, unrelated to this fix. + domainPat = regexp.MustCompile(`\b([a-zA-Z0-9][a-zA-Z0-9._-]*auth0(?:app)?\.com)\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/auth0oauth/auth0oauth_test.go b/pkg/detectors/auth0oauth/auth0oauth_test.go index 1f788240565a..ffdc3eae3147 100644 --- a/pkg/detectors/auth0oauth/auth0oauth_test.go +++ b/pkg/detectors/auth0oauth/auth0oauth_test.go @@ -50,13 +50,54 @@ func TestAuth0oAuth_Pattern(t *testing.T) { name: "invalid pattern", input: ` # do not share these credentials - auth0_credentials file: + auth0_credentials file: auth0_clientID: e4T9Cw1CtwE8ufoESVBB7Hi1U-e4T9Cw1CtwE8ufoESVBB7Hi1U - secret: MBqIYgxH0vZaL1s5314lgPDLqHX^ZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_+tTx1kCXOZY_JUy-rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS + secret: MBqIYgxH0vZaL1s5314lgPDLqHX^ZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_+tTx1kCXOZY_JUy-rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS domain: 9-KhTIdSopSaMQ2v1YxdFEJN#qd51AzSGQu2yRaFauth1.com `, want: nil, }, + { + // Auth0 Private Cloud tenants default to an Auth0-managed domain in + // the form *.auth0app.com rather than *.auth0.com. + name: "valid pattern - private cloud auth0app.com domain", + input: ` + # do not share these credentials + auth0_credentials file: + auth0_clientID: odJFCrnl2edlBDdz1C5Jau2RJtBRnlWmTSHf6pWk + secret: 6sHrFH2ZUCr-lgotu2iXW7GboIRoL3u6aHwnMztVuaP_coUNEhEkk_iqq8vH2BzNZV45pF + domain: api.example-corp.auth0app.com + `, + want: []string{"odJFCrnl2edlBDdz1C5Jau2RJtBRnlWmTSHf6pWk6sHrFH2ZUCr-lgotu2iXW7GboIRoL3u6aHwnMztVuaP_coUNEhEkk_iqq8vH2BzNZV45pF"}, + }, + { + // word-boundary behavior is unchanged by this fix: a leading run of + // word characters immediately before "auth0app.com" still matches in + // full, same as it already does today for "...auth0.com" domains. + name: "valid pattern - auth0app.com with no separator before suffix", + input: ` + # do not share these credentials + auth0_credentials file: + auth0_clientID: tyLBhhOhg9uhkxiiEZpFfk1OHAOEHYqM6Ojb + secret: 6mjBHqSiFVKu4MbMnrHontIKARAH_Ggl2JfaQqHu42bojteVs3qfNUfTAFnT0tEuw0dwQ0FIunWe8Cz6 + domain: xauth0app.com + `, + want: []string{"tyLBhhOhg9uhkxiiEZpFfk1OHAOEHYqM6Ojb6mjBHqSiFVKu4MbMnrHontIKARAH_Ggl2JfaQqHu42bojteVs3qfNUfTAFnT0tEuw0dwQ0FIunWe8Cz6"}, + }, + { + // a domain with neither the "auth0.com" nor "auth0app.com" suffix + // must not match — the optional (?:app)? group must not degrade the + // pattern into matching arbitrary ".com" domains. + name: "invalid pattern - domain has neither auth0.com nor auth0app.com suffix", + input: ` + # do not share these credentials + auth0_credentials file: + auth0_clientID: SNDCdyZQJiJSZQdoHwHen3SO3oXyGf3azU3iQOpMN0PZ + secret: Lqy1WwMZaMKA3P744B8vkKQlENCzsdfF8j61yX-ZFsan2Cw7gFp6r7O425u85HFJ_EJ4jKEIQOkrtDXtBi10Q71hA1 + domain: api.example.com + `, + want: nil, + }, } for _, test := range tests { diff --git a/pkg/detectors/autodesk/autodesk.go b/pkg/detectors/autodesk/autodesk.go index 368943ff7902..0ae61b471085 100644 --- a/pkg/detectors/autodesk/autodesk.go +++ b/pkg/detectors/autodesk/autodesk.go @@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/autopilot/autopilot.go b/pkg/detectors/autopilot/autopilot.go index c893af85b4d3..53d79c6c2880 100644 --- a/pkg/detectors/autopilot/autopilot.go +++ b/pkg/detectors/autopilot/autopilot.go @@ -2,10 +2,11 @@ package autopilot import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("autopilotapikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/aws/access_keys/accesskey.go b/pkg/detectors/aws/access_keys/accesskey.go index ebbe1b563353..6ad1b249f779 100644 --- a/pkg/detectors/aws/access_keys/accesskey.go +++ b/pkg/detectors/aws/access_keys/accesskey.go @@ -306,8 +306,8 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str return true, extraData, nil } -func (s scanner) CleanResults(results []detectors.Result) []detectors.Result { - return aws.CleanResults(results) +func (s scanner) CleanResults(results []detectors.Result, verificationEnabled bool) []detectors.Result { + return aws.CleanResults(results, verificationEnabled) } func (s scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/aws/access_keys/accesskey_test.go b/pkg/detectors/aws/access_keys/accesskey_test.go index bf6cb4774106..1fa8ae99a461 100644 --- a/pkg/detectors/aws/access_keys/accesskey_test.go +++ b/pkg/detectors/aws/access_keys/accesskey_test.go @@ -45,6 +45,20 @@ func TestAWS_Pattern(t *testing.T) { `, want: []string{"AKIAWGXZ9OPDOWUJMZGI:v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63"}, }, + { + name: "valid pattern - multiple secrets", + input: ` + aws credentials{ + id: ABIAS9L8MS5IPHTZPPUQ + secret: v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63 + }, + aws credentials{ + id: ABIAS9L8MS5IPHTZPXUR + secret: v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondDD21 + } + `, + want: []string{"ABIAS9L8MS5IPHTZPPUQ:v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63", "ABIAS9L8MS5IPHTZPPUQ:v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondDD21", "ABIAS9L8MS5IPHTZPXUR:v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63", "ABIAS9L8MS5IPHTZPXUR:v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondDD21"}, + }, { name: "invalid pattern", input: ` diff --git a/pkg/detectors/aws/appsync/appsync.go b/pkg/detectors/aws/appsync/appsync.go new file mode 100644 index 000000000000..4d3abed0aeaa --- /dev/null +++ b/pkg/detectors/aws/appsync/appsync.go @@ -0,0 +1,172 @@ +package appsync + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + apiKeyPat = regexp.MustCompile( + `\b(da2-[a-z0-9]{26})\b`, + ) + + endpointPat = regexp.MustCompile( + `(https:\/\/[a-z0-9]{26}\.appsync-api\.[a-z0-9-]+\.amazonaws\.com)([^a-zA-Z0-9.-]|$)`, + ) +) + +func (s Scanner) Keywords() []string { + return []string{"da2-"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + keys := make(map[string]struct{}) + endpoints := make(map[string]struct{}) + + for _, m := range apiKeyPat.FindAllStringSubmatch(dataStr, -1) { + keys[m[1]] = struct{}{} + } + + for _, m := range endpointPat.FindAllStringSubmatch(dataStr, -1) { + normalizedEndpoint := m[1] + "/graphql" + endpoints[normalizedEndpoint] = struct{}{} + } + + for key := range keys { + for endpoint := range endpoints { + + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_AWSAppSync, + Raw: []byte(key), + RawV2: []byte(fmt.Sprintf("%s:%s", endpoint, key)), + SecretParts: map[string]string{ + "key": key, + "endpoint": endpoint, + }, + ExtraData: map[string]string{ + "base_url": endpoint, + }, + } + + if verify { + verified, verificationErr := verifyAppSyncKey( + ctx, + s.getClient(), + endpoint, + key, + ) + + result.SetVerificationError(verificationErr, key) + result.Verified = verified + } + + results = append(results, result) + } + } + + return +} + +func verifyAppSyncKey( + ctx context.Context, + client *http.Client, + endpoint string, + key string, +) (bool, error) { + + query := `{"query":"query { __typename }"}` + + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + endpoint, + bytes.NewBufferString(query), + ) + if err != nil { + return false, err + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-api-key", key) + + res, err := client.Do(req) + if err != nil { + return false, err + } + + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + // https://docs.aws.amazon.com/appsync/latest/APIReference/CommonErrors.html + switch res.StatusCode { + + case http.StatusOK: + return true, nil + + case http.StatusBadGateway: + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + // Appsync returns 502 with a specific error message in the body when the key is valid but has no schema defined, + // so we treat that as a valid key + if bytes.Contains(bodyBytes, []byte("No schema definition exists")) { + return true, nil + } + return false, fmt.Errorf("502 Bad Gateway: unexpected response") + + // Appsync return 401 for invalid keys + // Appsync return 403 for some time after the token is deleted + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_AWSAppSync +} + +func (s Scanner) Description() string { + return "AWS AppSync is a managed GraphQL service. This detector identifies exposed AppSync API keys." +} diff --git a/pkg/detectors/aws/appsync/appsync_integration_test.go b/pkg/detectors/aws/appsync/appsync_integration_test.go new file mode 100644 index 000000000000..4be9541414dd --- /dev/null +++ b/pkg/detectors/aws/appsync/appsync_integration_test.go @@ -0,0 +1,201 @@ +//go:build detectors +// +build detectors + +package appsync + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestAppSync_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // Secrets are expected to be stored similarly to other detector tests + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeKey := testSecrets.MustGetField("APPSYNC_API_KEY") + endpoint := testSecrets.MustGetField("APPSYNC_API_URL") + + revokedEndpoint := "https://nr2nchyfwvc53lgrlvsa2pfpzq.appsync-api.us-east-1.amazonaws.com/graphql" + inactiveKey := "da2-abcdefghijklmnopqrstuvwxyz" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "endpoint=%s key=%s", endpoint, activeKey), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_AWSAppSync, + Verified: true, + Raw: []byte(activeKey), + RawV2: []byte(fmt.Sprintf("%s:%s", endpoint, activeKey)), + }, + }, + }, + { + name: "found, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "endpoint=%s key=%s", endpoint, activeKey), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_AWSAppSync, + Verified: false, + Raw: []byte(activeKey), + RawV2: []byte(fmt.Sprintf("%s:%s", endpoint, activeKey)), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, verification error unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "endpoint=%s key=%s", endpoint, activeKey), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_AWSAppSync, + Verified: false, + Raw: []byte(activeKey), + RawV2: []byte(fmt.Sprintf("%s:%s", endpoint, activeKey)), + }, + }, + wantVerificationErr: true, + }, + // Host will be unreachable for such cases + { + name: "found, Revoked key and endpoint", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "endpoint=%s key=%s", revokedEndpoint, inactiveKey), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_AWSAppSync, + Verified: false, + Raw: []byte(inactiveKey), + RawV2: []byte(fmt.Sprintf("%s:%s", revokedEndpoint, inactiveKey)), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, valid endpoint and invalid key", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "endpoint=%s key=%s", endpoint, inactiveKey), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_AWSAppSync, + Verified: false, + Raw: []byte(inactiveKey), + RawV2: []byte(fmt.Sprintf("%s:%s", endpoint, inactiveKey)), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("AppSync.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AppSync.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkAppSync_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/aws/appsync/appsync_test.go b/pkg/detectors/aws/appsync/appsync_test.go new file mode 100644 index 000000000000..3002b0ecf779 --- /dev/null +++ b/pkg/detectors/aws/appsync/appsync_test.go @@ -0,0 +1,136 @@ +package appsync + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestAppSync_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - key and endpoint", + input: ` + const endpoint = "https://abcdefghijklmnopqrstuvwxyz.appsync-api.us-east-1.amazonaws.com/graphql"; + const key = "da2-abcdefghijklmnopqrstuvwxyz"; + `, + want: []string{ + "https://abcdefghijklmnopqrstuvwxyz.appsync-api.us-east-1.amazonaws.com/graphql:da2-abcdefghijklmnopqrstuvwxyz", + }, + }, + { + name: "valid pattern - key and endpoint(endpoint without /graphql)", + input: ` + const endpoint = "https://abcdefghijklmnopqrstuvwxyz.appsync-api.us-east-1.amazonaws.com"; + const key = "da2-abcdefghijklmnopqrstuvwxyz"; + `, + want: []string{ + "https://abcdefghijklmnopqrstuvwxyz.appsync-api.us-east-1.amazonaws.com/graphql:da2-abcdefghijklmnopqrstuvwxyz", + }, + }, + { + name: "valid pattern - multiple keys and endpoints", + input: ` + https://aaaaaaaaaaaaaaaaaaaaaaaaaa.appsync-api.us-west-2.amazonaws.com/graphql + da2-aaaaaaaaaaaaaaaaaaaaaaaaaa + + https://bbbbbbbbbbbbbbbbbbbbbbbbbb.appsync-api.eu-central-1.amazonaws.com/graphql + da2-bbbbbbbbbbbbbbbbbbbbbbbbbb + `, + want: []string{ + "https://aaaaaaaaaaaaaaaaaaaaaaaaaa.appsync-api.us-west-2.amazonaws.com/graphql:da2-aaaaaaaaaaaaaaaaaaaaaaaaaa", + "https://bbbbbbbbbbbbbbbbbbbbbbbbbb.appsync-api.eu-central-1.amazonaws.com/graphql:da2-aaaaaaaaaaaaaaaaaaaaaaaaaa", + "https://aaaaaaaaaaaaaaaaaaaaaaaaaa.appsync-api.us-west-2.amazonaws.com/graphql:da2-bbbbbbbbbbbbbbbbbbbbbbbbbb", + "https://bbbbbbbbbbbbbbbbbbbbbbbbbb.appsync-api.eu-central-1.amazonaws.com/graphql:da2-bbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + }, + { + name: "invalid pattern - uppercase key", + input: ` + https://abcdefghijklmnopqrstuvwxyz.appsync-api.us-east-1.amazonaws.com/graphql + da2-ABCDEFGHIJKLMNOPQRSTUVWXYZ + `, + want: nil, + }, + { + name: "invalid pattern - key too short", + input: ` + https://abcdefghijklmnopqrstuvwxyz.appsync-api.us-east-1.amazonaws.com/graphql + da2-abc123 + `, + want: nil, + }, + { + name: "invalid pattern - key only", + input: ` + da2-abcdefghijklmnopqrstuvwxyz + `, + want: nil, + }, + { + name: "invalid pattern - malformed endpoint", + input: ` + https://abc.appsync-api.us-east-1.amazonaws.com/graphql + da2-abcdefghijklmnopqrstuvwxyz + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/aws/session_keys/sessionkey.go b/pkg/detectors/aws/session_keys/sessionkey.go index 2fe5bb5c90a9..990b3d5af058 100644 --- a/pkg/detectors/aws/session_keys/sessionkey.go +++ b/pkg/detectors/aws/session_keys/sessionkey.go @@ -330,13 +330,13 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str } } -func (s scanner) CleanResults(results []detectors.Result) []detectors.Result { - return aws.CleanResults(results) +func (s scanner) CleanResults(results []detectors.Result, verificationEnabled bool) []detectors.Result { + return aws.CleanResults(results, verificationEnabled) } // Reference: https://nitter.poast.org/TalBeerySec/status/1816449053841838223#m func checkSessionToken(sessionToken string, secret string) bool { - if !(strings.Contains(sessionToken, "YXdz") || strings.Contains(sessionToken, "Jb3JpZ2luX2Vj")) || + if (!strings.Contains(sessionToken, "YXdz") && !strings.Contains(sessionToken, "Jb3JpZ2luX2Vj")) || strings.Contains(sessionToken, secret) { // Handle error if the sessionToken is not a valid base64 string return false diff --git a/pkg/detectors/aws/session_keys/sessionkeys_test.go b/pkg/detectors/aws/session_keys/sessionkeys_test.go index b884dcb3e8f2..1e732016c897 100644 --- a/pkg/detectors/aws/session_keys/sessionkeys_test.go +++ b/pkg/detectors/aws/session_keys/sessionkeys_test.go @@ -46,6 +46,31 @@ func TestAWSSessionKey_Pattern(t *testing.T) { `, want: []string{"ASIABBKK02W42Q3IPSPG:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa="}, }, + { + name: "valid pattern - multiple secrets", + input: ` + aws credentials{ + id: ASIABBKK02W42Q3IPSPG + secret: fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O + session: aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa= + } + aws credentials{ + id: ASIABBKK02W42Q3IPBAR + secret: fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcMUFC + session: aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkabku= + } + `, + want: []string{ + "ASIABBKK02W42Q3IPSPG:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa=", + "ASIABBKK02W42Q3IPSPG:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkabku=", + "ASIABBKK02W42Q3IPSPG:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcMUFC:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa=", + "ASIABBKK02W42Q3IPSPG:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcMUFC:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkabku=", + "ASIABBKK02W42Q3IPBAR:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa=", + "ASIABBKK02W42Q3IPBAR:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkabku=", + "ASIABBKK02W42Q3IPBAR:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcMUFC:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa=", + "ASIABBKK02W42Q3IPBAR:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcMUFC:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkabku=", + }, + }, { name: "invalid pattern", input: ` diff --git a/pkg/detectors/aws/utils.go b/pkg/detectors/aws/utils.go index 468562b557b2..596a7752394b 100644 --- a/pkg/detectors/aws/utils.go +++ b/pkg/detectors/aws/utils.go @@ -86,11 +86,13 @@ func GetHMAC(key []byte, data []byte) []byte { return hasher.Sum(nil) } -func CleanResults(results []detectors.Result) []detectors.Result { +func CleanResults(results []detectors.Result, verificationEnabled bool) []detectors.Result { if len(results) == 0 { return results } - + if !verificationEnabled { + return results + } // For every ID, we want at most one result, preferably verified. idResults := map[string]detectors.Result{} for _, result := range results { diff --git a/pkg/detectors/azure_batch/azurebatch.go b/pkg/detectors/azure_batch/azurebatch.go index 56df04edbba5..32d3f803711f 100644 --- a/pkg/detectors/azure_batch/azurebatch.go +++ b/pkg/detectors/azure_batch/azurebatch.go @@ -116,7 +116,7 @@ func verifyMatch(ctx context.Context, client *http.Client, endpoint, accountName } return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/azure_cosmosdb/azure_cosmosdb.go b/pkg/detectors/azure_cosmosdb/azure_cosmosdb.go index e453c75ac6b6..79acb06c017b 100644 --- a/pkg/detectors/azure_cosmosdb/azure_cosmosdb.go +++ b/pkg/detectors/azure_cosmosdb/azure_cosmosdb.go @@ -7,10 +7,11 @@ import ( "encoding/base64" "errors" "fmt" + regexp "github.com/wasilibs/go-re2" "io" + "net" "net/http" "net/url" - "regexp" "strings" "time" @@ -74,11 +75,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for key := range uniqueKeyMatches { for accountUrl := range uniqueAccountMatches { - if invalidHosts.Exists(accountUrl) { - delete(uniqueAccountMatches, accountUrl) - continue - } - s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_AzureCosmosDBKeyIdentifiable, Raw: []byte(key), @@ -90,30 +86,43 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result ExtraData: map[string]string{}, } - if verify { - var verified bool - var verificationErr error - - client := s.getClient() + // The database type follows from the account URL alone, so it is recorded for every + // finding rather than only for the ones that reach a verification request. + if strings.Contains(accountUrl, ".documents.azure.com") { + s1.ExtraData["DB Type"] = "Document" + } else if strings.Contains(accountUrl, ".table.cosmos.azure.com") { + s1.ExtraData["DB Type"] = "Table" + } - // perform verification based on db type - if strings.Contains(accountUrl, ".documents.azure.com") { - verified, verificationErr = verifyCosmosDocumentDB(client, accountUrl, key) - s1.ExtraData["DB Type"] = "Document" + if verify { + if invalidHosts.Exists(accountUrl) { + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + s1.SetVerificationError(errNoHost) + } else { + var verified bool + var verificationErr error + + client := s.getClient() + + // perform verification based on db type + if strings.Contains(accountUrl, ".documents.azure.com") { + verified, verificationErr = verifyCosmosDocumentDB(client, accountUrl, key) + } else if strings.Contains(accountUrl, ".table.cosmos.azure.com") { + verified, verificationErr = verifyCosmosTableDB(client, accountUrl, key) + } - } else if strings.Contains(accountUrl, ".table.cosmos.azure.com") { - verified, verificationErr = verifyCosmosTableDB(client, accountUrl, key) - s1.ExtraData["DB Type"] = "Table" - } + s1.Verified = verified + if verificationErr != nil { + var dnsErr *net.DNSError + if errors.As(verificationErr, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(accountUrl, struct{}{}) + } - s1.Verified = verified - if verificationErr != nil { - if errors.Is(verificationErr, errNoHost) { - invalidHosts.Set(accountUrl, struct{}{}) - continue + s1.SetVerificationError(verificationErr) } - - s1.SetVerificationError(verificationErr) } } @@ -148,11 +157,6 @@ func verifyCosmosDocumentDB(client *http.Client, accountUrl, key string) (bool, resp, err := client.Do(req) if err != nil { - // lookup foo.documents.azure.com: no such host - if strings.Contains(err.Error(), "no such host") { - return false, errNoHost - } - return false, err } defer func() { diff --git a/pkg/detectors/azure_cosmosdb/table.go b/pkg/detectors/azure_cosmosdb/table.go index 157b205c4dda..8eed03f8b24c 100644 --- a/pkg/detectors/azure_cosmosdb/table.go +++ b/pkg/detectors/azure_cosmosdb/table.go @@ -38,11 +38,6 @@ func verifyCosmosTableDB(client *http.Client, accountUrl, key string) (bool, err resp, err := client.Do(req) if err != nil { - // lookup foo.table.cosmos.azure.com: no such host - if strings.Contains(err.Error(), "no such host") { - return false, errNoHost - } - return false, err } defer func() { diff --git a/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go b/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go index fa4041cf28ad..a379a673c96a 100644 --- a/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go +++ b/pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go @@ -3,8 +3,8 @@ package v2 import ( "context" "errors" + regexp "github.com/wasilibs/go-re2" "net/http" - "regexp" "strings" "github.com/trufflesecurity/trufflehog/v3/pkg/common" diff --git a/pkg/detectors/azureapimanagement/repositorykey/repositorykey.go b/pkg/detectors/azureapimanagement/repositorykey/repositorykey.go index 403a519c8526..57f9e2d25ff9 100644 --- a/pkg/detectors/azureapimanagement/repositorykey/repositorykey.go +++ b/pkg/detectors/azureapimanagement/repositorykey/repositorykey.go @@ -30,7 +30,7 @@ var ( passwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "password"}) + `\b(git&[0-9]{12}&[a-zA-Z0-9\/+]{85}[a-zA-Z0-9]==)`) invalidHosts = simple.NewCache[struct{}]() - noSuchHostErr = errors.New("Could not resolve host") + errNoSuchHost = errors.New("no such host") ) const ( @@ -60,7 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result uniquePasswordMatches[strings.TrimSpace(matches[1])] = struct{}{} } -EndpointLoop: for urlMatch := range uniqueUrlsMatches { for passwordMatch := range uniquePasswordMatches { s1 := detectors.Result{ @@ -75,18 +74,21 @@ EndpointLoop: if verify { if invalidHosts.Exists(urlMatch) { - logger.V(3).Info("Skipping invalid registry", "url", urlMatch) - continue EndpointLoop - } - - isVerified, err := verifyUrlPassword(ctx, urlMatch, azureGitUsername, passwordMatch) - s1.Verified = isVerified - if err != nil { - if errors.Is(err, noSuchHostErr) { - invalidHosts.Set(urlMatch, struct{}{}) - continue EndpointLoop + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + logger.V(3).Info("Skipping verification: no such host", "url", urlMatch) + s1.SetVerificationError(errNoSuchHost, urlMatch) + } else { + isVerified, err := verifyUrlPassword(ctx, urlMatch, azureGitUsername, passwordMatch) + s1.Verified = isVerified + if err != nil { + if errors.Is(err, errNoSuchHost) { + invalidHosts.Set(urlMatch, struct{}{}) + } + s1.SetVerificationError(err, urlMatch) } - s1.SetVerificationError(err, urlMatch) } } results = append(results, s1) @@ -156,7 +158,7 @@ func verifyUrlPassword(_ context.Context, repoUrl, user, password string) (bool, if strings.Contains(outputString, "Authentication failed") { return false, nil } else if strings.Contains(outputString, "Could not resolve host") { - return false, noSuchHostErr + return false, errNoSuchHost } return false, err } diff --git a/pkg/detectors/azureapimanagementsubscriptionkey/azureapimanagementsubscriptionkey.go b/pkg/detectors/azureapimanagementsubscriptionkey/azureapimanagementsubscriptionkey.go index 97841c92a33f..29b1fb8f381f 100644 --- a/pkg/detectors/azureapimanagementsubscriptionkey/azureapimanagementsubscriptionkey.go +++ b/pkg/detectors/azureapimanagementsubscriptionkey/azureapimanagementsubscriptionkey.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "net/http" "strings" @@ -31,7 +32,7 @@ var ( keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", ".azure-api.net", "subscription", "key"}) + `([a-zA-Z-0-9]{32})`) // pattern for both Primary and secondary key invalidHosts = simple.NewCache[struct{}]() - noSuchHostErr = errors.New("no such host") + errNoSuchHost = errors.New("no such host") ) // Keywords are used for efficiently pre-filtering chunks. @@ -54,7 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result keyMatchesUnique[strings.TrimSpace(keyMatch[1])] = struct{}{} } -EndpointLoop: for baseUrl := range urlMatchesUnique { for key := range keyMatchesUnique { s1 := detectors.Result{ @@ -69,23 +69,27 @@ EndpointLoop: if verify { if invalidHosts.Exists(baseUrl) { - logger.V(3).Info("Skipping invalid registry", "baseUrl", baseUrl) - continue EndpointLoop - } - - client := s.client - if client == nil { - client = defaultClient - } + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + logger.V(3).Info("Skipping verification: no such host", "baseUrl", baseUrl) + s1.SetVerificationError(errNoSuchHost, baseUrl) + } else { + client := s.client + if client == nil { + client = defaultClient + } - isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, key) - s1.Verified = isVerified - if verificationErr != nil { - if errors.Is(verificationErr, noSuchHostErr) { - invalidHosts.Set(baseUrl, struct{}{}) - continue EndpointLoop + isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, key) + s1.Verified = isVerified + if verificationErr != nil { + var dnsErr *net.DNSError + if errors.As(verificationErr, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(baseUrl, struct{}{}) + } + s1.SetVerificationError(verificationErr, baseUrl) } - s1.SetVerificationError(verificationErr, baseUrl) } } @@ -118,9 +122,11 @@ func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, baseUrl, req.Header.Set("Ocp-Apim-Subscription-Key", key) resp, err := client.Do(req) if err != nil { - return false, nil + // A transport failure means the credential was never tested, so the result is + // indeterminate. Returning (false, nil) here would report it as a rejection. + return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/azureappconfigconnectionstring/azureappconfigconnectionstring.go b/pkg/detectors/azureappconfigconnectionstring/azureappconfigconnectionstring.go index 0beecd260590..bce5cbb6fb09 100644 --- a/pkg/detectors/azureappconfigconnectionstring/azureappconfigconnectionstring.go +++ b/pkg/detectors/azureappconfigconnectionstring/azureappconfigconnectionstring.go @@ -153,7 +153,7 @@ func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, endpoint, if err != nil { return false, fmt.Errorf("request failed: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check the response status switch resp.StatusCode { diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go index e2556eeb8858..bfa8282b777b 100644 --- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go +++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" "io" + "net" "net/http" - "strings" regexp "github.com/wasilibs/go-re2" @@ -73,7 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result passwordMatches[p] = struct{}{} } -EndpointLoop: for username := range registryMatches { for password := range passwordMatches { r := detectors.Result{ @@ -89,26 +88,30 @@ EndpointLoop: if verify { if invalidHosts.Exists(username) { - logger.V(3).Info("Skipping invalid registry", "username", username) - continue EndpointLoop - } - - client := s.client - if client == nil { - client = defaultClient - } + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + logger.V(3).Info("Skipping verification: no such host", "username", username) + r.SetVerificationError(errNoSuchHost, password) + } else { + client := s.client + if client == nil { + client = defaultClient + } - isVerified, verificationErr := verifyMatch(ctx, client, username, password) - if isVerified { - delete(passwordMatches, password) - r.Verified = true - } - if verificationErr != nil { - if errors.Is(verificationErr, noSuchHostErr) { - invalidHosts.Set(username, struct{}{}) - continue EndpointLoop + isVerified, verificationErr := verifyMatch(ctx, client, username, password) + if isVerified { + delete(passwordMatches, password) + r.Verified = true + } + if verificationErr != nil { + var dnsErr *net.DNSError + if errors.As(verificationErr, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(username, struct{}{}) + } + r.SetVerificationError(verificationErr, password) } - r.SetVerificationError(verificationErr, password) } } @@ -126,7 +129,7 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) { return false, "" } -var noSuchHostErr = errors.New("no such host") +var errNoSuchHost = errors.New("no such host") func verifyMatch(ctx context.Context, client *http.Client, username string, password string) (bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s.azurecr.io/v2/", username), nil) @@ -137,10 +140,6 @@ func verifyMatch(ctx context.Context, client *http.Client, username string, pass req.SetBasicAuth(username, password) res, err := client.Do(req) if err != nil { - // lookup foo.azurecr.io: no such host - if strings.Contains(err.Error(), "no such host") { - return false, noSuchHostErr - } return false, err } defer func() { diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go index b020c72dce37..edf1b03e125f 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go @@ -68,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth("", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() hasVerifiedRes, _ := common.ResponseContainsSubstring(res.Body, "lastUpdateTime") if res.StatusCode >= 200 && res.StatusCode < 300 && hasVerifiedRes { s1.Verified = true diff --git a/pkg/detectors/azuredirectmanagementkey/azuredirectmanagementkey.go b/pkg/detectors/azuredirectmanagementkey/azuredirectmanagementkey.go index b03481486b53..2290a0ce189f 100644 --- a/pkg/detectors/azuredirectmanagementkey/azuredirectmanagementkey.go +++ b/pkg/detectors/azuredirectmanagementkey/azuredirectmanagementkey.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "errors" "fmt" + "net" "net/http" "strings" "time" @@ -37,7 +38,7 @@ var ( keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", ".management.azure-api.net"}) + `([a-zA-Z0-9+\/]{83,85}[a-zA-Z0-9]==)`) // pattern for both Primary and secondary key invalidHosts = simple.NewCache[struct{}]() - noSuchHostErr = errors.New("no such host") + errNoSuchHost = errors.New("no such host") ) // Keywords are used for efficiently pre-filtering chunks. @@ -60,7 +61,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result keyMatchesUnique[strings.TrimSpace(keyMatch[1])] = struct{}{} } -EndpointLoop: for baseUrl, serviceName := range urlMatchesUnique { for key := range keyMatchesUnique { s1 := detectors.Result{ @@ -75,23 +75,27 @@ EndpointLoop: if verify { if invalidHosts.Exists(baseUrl) { - logger.V(3).Info("Skipping invalid registry", "baseUrl", baseUrl) - continue EndpointLoop - } - - client := s.client - if client == nil { - client = defaultClient - } + // An earlier candidate already proved this host does not resolve, so the lookup is + // skipped. The finding is still reported: dropping it here would make the reported + // secrets depend on whether verification was requested, and on the order in which + // candidates happened to be processed. + logger.V(3).Info("Skipping verification: no such host", "baseUrl", baseUrl) + s1.SetVerificationError(errNoSuchHost, baseUrl) + } else { + client := s.client + if client == nil { + client = defaultClient + } - isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, serviceName, key) - s1.Verified = isVerified - if verificationErr != nil { - if errors.Is(verificationErr, noSuchHostErr) { - invalidHosts.Set(baseUrl, struct{}{}) - continue EndpointLoop + isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, serviceName, key) + s1.Verified = isVerified + if verificationErr != nil { + var dnsErr *net.DNSError + if errors.As(verificationErr, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(baseUrl, struct{}{}) + } + s1.SetVerificationError(verificationErr, baseUrl) } - s1.SetVerificationError(verificationErr, baseUrl) } } @@ -131,9 +135,11 @@ func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, baseUrl, req.Header.Set("Authorization", fmt.Sprintf("SharedAccessSignature %s", accessToken)) resp, err := client.Do(req) if err != nil { - return false, nil + // A transport failure means the credential was never tested, so the result is + // indeterminate. Returning (false, nil) here would report it as a rejection. + return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey.go b/pkg/detectors/azurefunctionkey/azurefunctionkey.go index 9a93494b1eb5..2bf02b877a03 100644 --- a/pkg/detectors/azurefunctionkey/azurefunctionkey.go +++ b/pkg/detectors/azurefunctionkey/azurefunctionkey.go @@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/azuresastoken/azuresastoken.go b/pkg/detectors/azuresastoken/azuresastoken.go index a887a9ed3150..6612083747ca 100644 --- a/pkg/detectors/azuresastoken/azuresastoken.go +++ b/pkg/detectors/azuresastoken/azuresastoken.go @@ -31,13 +31,33 @@ var ( // microsoft storage resource naming rules: https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage:~:text=format%3A%0AVaultName_KeyName_KeyVersion.-,Microsoft.Storage,-Expand%20table urlPat = regexp.MustCompile(`https://([a-zA-Z0-9][a-z0-9_-]{1,22}[a-zA-Z0-9])\.blob\.core\.windows\.net/[a-z0-9]([a-z0-9-]{1,61}[a-z0-9])?(?:/[a-zA-Z0-9._-]+)*`) - keyPat = regexp.MustCompile( - detectors.PrefixRegex([]string{"azure", "sas", "token", "blob", ".blob.core.windows.net"}) + - `(sp=[racwdli]+&st=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z&se=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z(?:&sip=\d{1,3}(?:\.\d{1,3}){3}(?:-\d{1,3}(?:\.\d{1,3}){3})?)?(&spr=https)?(?:,https)?&sv=\d{4}-\d{2}-\d{2}&sr=[bcfso]&sig=[a-zA-Z0-9%]{10,})`) + // A SAS token is a query string of `&`-joined `key=value` pairs. The parameters + // can appear in any order, and values may be URL-encoded (e.g. Azure Storage + // Explorer encodes the `:` in the timestamps as %3A and the `+`, `/`, `=` in the + // signature as %2B, %2F, %3D). Rather than enumerate every permutation in one + // regex, match a contiguous run of query parameters and validate the SAS-specific + // parameters in keyMatchIsSASToken below. This keeps detection order-independent. + // The value class excludes `=` so a SAS token preceded by a short lowercase + // `key=` (e.g. `sas=sp=r&...`) is not absorbed from that preceding key, which + // would bury the real `sp` parameter inside the first value and fail + // validation. Azure URL-encodes any `=` inside a value (the signature padding + // becomes %3D), so excluding the literal does not drop a legitimate value. + sasQueryPat = regexp.MustCompile(`[a-z]{2,4}=[^&=\s"'<>]+(?:&[a-z]{2,4}=[^&=\s"'<>]+)+`) + + // Validators for the individual SAS parameters. The `sp` permission set and `sr` + // resource set match the formats the previous, order-locked regex accepted. + spValuePat = regexp.MustCompile(`^[racwdli]+$`) + svValuePat = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) + srValuePat = regexp.MustCompile(`^[bcfso]$`) + // The `:` separators may be URL-encoded; percent-encoding hex digits are + // case-insensitive per RFC 3986, so accept both %3A and %3a. + timeValuePat = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}(?::|%3[Aa])\d{2}(?::|%3[Aa])\d{2}Z$`) + sigValuePat = regexp.MustCompile(`^[a-zA-Z0-9%+/=]{10,}$`) + sipValuePat = regexp.MustCompile(`^\d{1,3}(?:\.\d{1,3}){3}(?:-\d{1,3}(?:\.\d{1,3}){3})?$`) invalidStorageAccounts = simple.NewCache[struct{}]() - noSuchHostErr = errors.New("no such host") + errNoSuchHost = errors.New("no such host") ) func (s Scanner) Keywords() []string { @@ -55,6 +75,60 @@ func (s Scanner) Description() string { return "An Azure Shared Access Signature (SAS) token is a time-limited, permission-based URL query string that grants secure, granular access to Azure Storage resources (e.g., blobs, containers, files) without exposing account keys." } +// keyMatchIsSASToken reports whether a matched query-parameter run is a Azure +// Storage SAS token. The required parameters may appear in any order; values may +// be URL-encoded. This reproduces the validations the previous order-locked regex +// enforced (permission set, resource type, timestamp shape, signature, optional IP) +// without depending on parameter order. +func keyMatchIsSASToken(query string) bool { + params := make(map[string]string) + for _, pair := range strings.Split(query, "&") { + key, value, ok := strings.Cut(pair, "=") + if !ok { + continue + } + params[key] = value + } + + sp, ok := params["sp"] + if !ok || !spValuePat.MatchString(sp) { + return false + } + sv, ok := params["sv"] + if !ok || !svValuePat.MatchString(sv) { + return false + } + sr, ok := params["sr"] + if !ok || !srValuePat.MatchString(sr) { + return false + } + sig, ok := params["sig"] + if !ok || !sigValuePat.MatchString(sig) { + return false + } + + // A SAS token carries a start time, an expiry time, or both; any present one + // must be well-formed. + st, hasStart := params["st"] + se, hasExpiry := params["se"] + if !hasStart && !hasExpiry { + return false + } + if hasStart && !timeValuePat.MatchString(st) { + return false + } + if hasExpiry && !timeValuePat.MatchString(se) { + return false + } + + // The IP restriction is optional, but must be a valid address or range if set. + if sip, ok := params["sip"]; ok && !sipValuePat.MatchString(sip) { + return false + } + + return true +} + func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { logger := logContext.AddLogger(ctx).Logger().WithName("azuresas") @@ -68,8 +142,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // deduplicate keyMatches keyMatchesUnique := make(map[string]struct{}) - for _, keyMatch := range keyPat.FindAllStringSubmatch(dataStr, -1) { - keyMatchesUnique[keyMatch[1]] = struct{}{} + for _, keyMatch := range sasQueryPat.FindAllString(dataStr, -1) { + if keyMatchIsSASToken(keyMatch) { + keyMatchesUnique[keyMatch] = struct{}{} + } } // Check results. @@ -101,7 +177,7 @@ UrlLoop: s1.Verified = isVerified if verificationErr != nil { - if errors.Is(verificationErr, noSuchHostErr) { + if errors.Is(verificationErr, errNoSuchHost) { invalidStorageAccounts.Set(storageAccount, struct{}{}) continue UrlLoop } @@ -131,11 +207,11 @@ func verifyMatch(ctx context.Context, client *http.Client, url, key string, retr res, err := client.Do(req) if err != nil { if strings.Contains(err.Error(), "no such host") { - return false, noSuchHostErr + return false, errNoSuchHost } return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() bodyBytes, err := io.ReadAll(res.Body) if err != nil { diff --git a/pkg/detectors/azuresastoken/azuresastoken_test.go b/pkg/detectors/azuresastoken/azuresastoken_test.go index fec99debde16..59de2623dd70 100644 --- a/pkg/detectors/azuresastoken/azuresastoken_test.go +++ b/pkg/detectors/azuresastoken/azuresastoken_test.go @@ -60,6 +60,43 @@ func TestAzureSASToken_Pattern(t *testing.T) { `, want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecurity/test_blob.txtsp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D"}, }, + { + // Storage Explorer emits the parameters in a different order (sv first, + // sp last) and URL-encodes the ':' in the timestamps as %3A. See #4732. + name: "valid pattern, alternate order with url-encoded timestamps", + input: ` + https://accountname.blob.core.windows.net/sorted?sv=2023-01-03&st=2025-06-18T08%3A45%3A11Z&se=2025-06-19T08%3A45%3A11Z&sr=c&sp=r&sig=ow2a1XbXmD4%2BEv9LBUkek8R%2FrAjvrQFpenUbzztILn8%3D + `, + want: []string{"https://accountname.blob.core.windows.net/sortedsv=2023-01-03&st=2025-06-18T08%3A45%3A11Z&se=2025-06-19T08%3A45%3A11Z&sr=c&sp=r&sig=ow2a1XbXmD4%2BEv9LBUkek8R%2FrAjvrQFpenUbzztILn8%3D"}, + }, + { + // Percent-encoding hex digits are case-insensitive (RFC 3986), so a + // timestamp encoded with lowercase %3a must match just like %3A. + name: "valid pattern, url-encoded timestamps with lowercase hex", + input: ` + https://accountname.blob.core.windows.net/sorted?sv=2023-01-03&st=2025-06-18T08%3a45%3a11Z&se=2025-06-19T08%3a45%3a11Z&sr=c&sp=r&sig=ow2a1XbXmD4%2BEv9LBUkek8R%2FrAjvrQFpenUbzztILn8%3D + `, + want: []string{"https://accountname.blob.core.windows.net/sortedsv=2023-01-03&st=2025-06-18T08%3a45%3a11Z&se=2025-06-19T08%3a45%3a11Z&sr=c&sp=r&sig=ow2a1XbXmD4%2BEv9LBUkek8R%2FrAjvrQFpenUbzztILn8%3D"}, + }, + { + // A SAS token assigned to a short lowercase variable (`sas=sp=r&...`) + // must still be captured: the parameter run has to start at `sp`, not + // be absorbed from the preceding `sas=` key. + name: "valid pattern preceded by lowercase variable assignment", + input: ` + sas=sp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D + AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity + `, + want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecuritysp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D"}, + }, + { + name: "non-sas query string is ignored", + input: ` + AZURE_BLOB_QUERY=comp=list&restype=container&maxresults=100 + AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity + `, + want: nil, + }, { name: "invalid pattern", input: ` diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go index bf76892f9f80..65af1b9d3482 100644 --- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go +++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey.go @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 || res.StatusCode == 403 { diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go index 25a68e744344..e15905fcea29 100644 --- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go +++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 || res.StatusCode == 403 { diff --git a/pkg/detectors/bannerbear/v1/bannerbear_test.go b/pkg/detectors/bannerbear/v1/bannerbear_test.go index 009862310559..ffd623e7a7d3 100644 --- a/pkg/detectors/bannerbear/v1/bannerbear_test.go +++ b/pkg/detectors/bannerbear/v1/bannerbear_test.go @@ -39,7 +39,7 @@ func TestBannerBear_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"yvxpthLIcYpZweFpPOVeCOtt"}, @@ -77,7 +77,7 @@ func TestBannerBear_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bannerbear/v2/bannerbear_test.go b/pkg/detectors/bannerbear/v2/bannerbear_test.go index f75714a71753..b8ccbac0ecb1 100644 --- a/pkg/detectors/bannerbear/v2/bannerbear_test.go +++ b/pkg/detectors/bannerbear/v2/bannerbear_test.go @@ -39,7 +39,7 @@ func TestBannerBear_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"bb_pr_abcdc2b40ef44abcd8cbf3739aabcd"}, @@ -77,7 +77,7 @@ func TestBannerBear_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/baremetrics/baremetrics_test.go b/pkg/detectors/baremetrics/baremetrics_test.go index 71ae28b90411..aca628dcd4df 100644 --- a/pkg/detectors/baremetrics/baremetrics_test.go +++ b/pkg/detectors/baremetrics/baremetrics_test.go @@ -39,7 +39,7 @@ func TestBareMetrics_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"sk_nGDJWCkPiFAKE5XFTzUUA"}, @@ -77,7 +77,7 @@ func TestBareMetrics_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/beamer/beamer_test.go b/pkg/detectors/beamer/beamer_test.go index 2a9b710f05c6..d59867ccdc00 100644 --- a/pkg/detectors/beamer/beamer_test.go +++ b/pkg/detectors/beamer/beamer_test.go @@ -38,7 +38,7 @@ func TestBeamer_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"DyVdf7+cAXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO="}, @@ -75,7 +75,7 @@ func TestBeamer_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/beebole/beebole_test.go b/pkg/detectors/beebole/beebole_test.go index b9f668f06ecb..8ea8ba05b835 100644 --- a/pkg/detectors/beebole/beebole_test.go +++ b/pkg/detectors/beebole/beebole_test.go @@ -39,7 +39,7 @@ func TestBeeBole_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"bn6htprmfpukfalts4muwalxh9j15ucvnrfdme8t"}, @@ -77,7 +77,7 @@ func TestBeeBole_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/besnappy/besnappy_test.go b/pkg/detectors/besnappy/besnappy_test.go index 0ad2393210c1..0528d29259f7 100644 --- a/pkg/detectors/besnappy/besnappy_test.go +++ b/pkg/detectors/besnappy/besnappy_test.go @@ -39,7 +39,7 @@ func TestBeSnappy_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"f58c5d37d7876d32cfdd823f8fe4ded364a8d483b5dbfadcc55ad801b3be8523"}, @@ -77,7 +77,7 @@ func TestBeSnappy_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/besttime/besttime_test.go b/pkg/detectors/besttime/besttime_test.go index 3af4f58f1faa..06f4d71eb652 100644 --- a/pkg/detectors/besttime/besttime_test.go +++ b/pkg/detectors/besttime/besttime_test.go @@ -36,7 +36,7 @@ func TestBestTime_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"pri_099889f14d114dfaae476569b395eade"}, @@ -71,7 +71,7 @@ func TestBestTime_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/betterstack/betterstack_test.go b/pkg/detectors/betterstack/betterstack_test.go index 45ff9579bff9..b5d7e35b5e96 100644 --- a/pkg/detectors/betterstack/betterstack_test.go +++ b/pkg/detectors/betterstack/betterstack_test.go @@ -38,7 +38,7 @@ func TestBetterStack_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } func getBetterStackToken() string{ return "ntJD0ER8QpuT0O1WqsclApO2" } `, @@ -76,7 +76,7 @@ func TestBetterStack_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } func getBetterStackToken() string{ return "DyntJD0ER8QpuT0O1WqsclApO2" } `, diff --git a/pkg/detectors/billomat/billomat_test.go b/pkg/detectors/billomat/billomat_test.go index 0aa00263df0d..587e289ce55a 100644 --- a/pkg/detectors/billomat/billomat_test.go +++ b/pkg/detectors/billomat/billomat_test.go @@ -38,7 +38,7 @@ func TestBilloMat_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status if resp.StatusCode == http.StatusOK { diff --git a/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey.go b/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey.go deleted file mode 100644 index 77d3244ecb66..000000000000 --- a/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey.go +++ /dev/null @@ -1,96 +0,0 @@ -package bingsubscriptionkey - -import ( - "context" - "fmt" - "io" - "net/http" - - regexp "github.com/wasilibs/go-re2" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" -) - -type Scanner struct { - client *http.Client -} - -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = common.SaneHttpClient() - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bing"}) + `\b([a-fA-F0-9]{32})\b`) -) - -func (s Scanner) Keywords() []string { - return []string{"bing"} -} - -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - uniqueMatches := make(map[string]struct{}) - for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { - uniqueMatches[match[1]] = struct{}{} - } - - for match := range uniqueMatches { - s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_BingSubscriptionKey, - Raw: []byte(match), - SecretParts: map[string]string{"key": match}, - } - - if verify { - client := s.client - if client == nil { - client = defaultClient - } - - isVerified, verificationErr := verifyMatch(ctx, client, match) - s1.Verified = isVerified - s1.SetVerificationError(verificationErr, match) - } - - results = append(results, s1) - } - - return -} - -func verifyMatch(ctx context.Context, client *http.Client, subscriptionKey string) (bool, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.bing.microsoft.com/v7.0/search?q=trufflehog", nil) - if err != nil { - return false, err - } - - req.Header.Add("Ocp-Apim-Subscription-Key", subscriptionKey) - - res, err := client.Do(req) - if err != nil { - return false, err - } - defer func() { - _, _ = io.Copy(io.Discard, res.Body) - _ = res.Body.Close() - }() - - switch res.StatusCode { - case http.StatusOK: - return true, nil - case http.StatusUnauthorized: - return false, nil - default: - return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - } -} - -func (s Scanner) Type() detector_typepb.DetectorType { - return detector_typepb.DetectorType_BingSubscriptionKey -} - -func (s Scanner) Description() string { - return "Bing Subscription Key is a key used to access the Bing Web Search API." -} diff --git a/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey_test.go b/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey_test.go deleted file mode 100644 index 52a8a61d1751..000000000000 --- a/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package bingsubscriptionkey - -import ( - "context" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/require" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" -) - -func TestBingsubscriptionkey_Pattern(t *testing.T) { - d := Scanner{} - ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - tests := []struct { - name string - input string - want []string - }{ - { - name: "valid pattern", - input: ` - func main() { - url := "https://api.example.net/v2/api" - - // Create a new request with the secret as a header - req, err := http.NewRequest("GET", url, http.NoBody) - if err != nil { - fmt.Println("Error creating request:", err) - return - } - - // set bing subscription key - bingKey := "89017d414ed64edb9c776d4a52102b9a" - req.Header.Set("Ocp-Apim-Subscription-Key", bingKey) - - // Perform the request - client := &http.Client{} - resp, _ := client.Do(req) - defer resp.Body.Close() - }`, - want: []string{"89017d414ed64edb9c776d4a52102b9a"}, - }, - { - name: "valid pattern - xml", - input: ` - - GLOBAL - {bing} - {bing AQAAABAAA dB963b030A1DafB02d8299F04A00a306} - configuration for production - 2023-05-18T14:32:10Z - jenkins-admin - - `, - want: []string{"dB963b030A1DafB02d8299F04A00a306"}, - }, - { - name: "invalid pattern", - input: ` - func main() { - url := "https://api.example.net/v2/api" - - // Create a new request with the secret as a header - req, err := http.NewRequest("GET", url, http.NoBody) - if err != nil { - fmt.Println("Error creating request:", err) - return - } - - // set bing subscription key - bingKey := "89017d414ed64edb9c776d4J52102b9" - req.Header.Set("Ocp-Apim-Subscription-Key", bingKey) - - // Perform the request - client := &http.Client{} - resp, _ := client.Do(req) - defer resp.Body.Close() - }`, - want: []string{}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { - t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords()) - return - } - - results, err := d.FromData(context.Background(), false, []byte(test.input)) - require.NoError(t, err) - - if len(results) != len(test.want) { - t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results)) - return - } - - actual := make(map[string]struct{}, len(results)) - for _, r := range results { - if len(r.RawV2) > 0 { - actual[string(r.RawV2)] = struct{}{} - } else { - actual[string(r.Raw)] = struct{}{} - } - } - - expected := make(map[string]struct{}, len(test.want)) - for _, v := range test.want { - expected[v] = struct{}{} - } - - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) - } - }) - } -} diff --git a/pkg/detectors/bitbar/bitbar_test.go b/pkg/detectors/bitbar/bitbar_test.go index a598ef4aaf6d..6b643a7332ab 100644 --- a/pkg/detectors/bitbar/bitbar_test.go +++ b/pkg/detectors/bitbar/bitbar_test.go @@ -42,7 +42,7 @@ func TestBitBar_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"64pq66z15thg8fh3acd00l35lpyg7c82"}, @@ -83,7 +83,7 @@ func TestBitBar_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bitbucketapppassword/bitbucketapppassword.go b/pkg/detectors/bitbucketapppassword/bitbucketapppassword.go index 273dabcbd241..3f523d23fef7 100644 --- a/pkg/detectors/bitbucketapppassword/bitbucketapppassword.go +++ b/pkg/detectors/bitbucketapppassword/bitbucketapppassword.go @@ -4,9 +4,9 @@ import ( "context" "encoding/base64" "fmt" + regexp "github.com/wasilibs/go-re2" "io" "net/http" - "regexp" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" diff --git a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go index d92960ca6355..9f2499802e01 100644 --- a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go +++ b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go @@ -43,7 +43,7 @@ func TestBitCoinAverage_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"WZizqeWvRnhZmFlpc5pMc92NP1Du19wxxpd5zjsYY8F"}, @@ -85,7 +85,7 @@ func TestBitCoinAverage_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go index a79913f78481..4a17e192d977 100644 --- a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go +++ b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go @@ -39,7 +39,7 @@ func TestBitlyAccessToken_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"2xN7puShxzNf5fZleQthTg305lKr7KrbW95D3gSD"}, @@ -77,7 +77,7 @@ func TestBitlyAccessToken_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bitmex/bitmex_test.go b/pkg/detectors/bitmex/bitmex_test.go index 209e1afd253b..9e90b3bfc4e7 100644 --- a/pkg/detectors/bitmex/bitmex_test.go +++ b/pkg/detectors/bitmex/bitmex_test.go @@ -46,7 +46,7 @@ func TestBitmex_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"EPwUIxOIveS463D_2O9LFgkzW_HlMtrmELzXm4bSlWv49JLcgvg5hvu467WbbnpmoEA-RjrY"}, @@ -91,7 +91,7 @@ func TestBitmex_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/blazemeter/blazemeter_test.go b/pkg/detectors/blazemeter/blazemeter_test.go index 2f5d6f45d381..61362cd85d4c 100644 --- a/pkg/detectors/blazemeter/blazemeter_test.go +++ b/pkg/detectors/blazemeter/blazemeter_test.go @@ -39,7 +39,7 @@ func TestBlazeMeter_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"sjbuxa3m-vs4n-ykl8-8jpv-i09hdidciolp"}, @@ -77,7 +77,7 @@ func TestBlazeMeter_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/blitapp/blitapp_test.go b/pkg/detectors/blitapp/blitapp_test.go index a759f7949e69..fe0a86aac375 100644 --- a/pkg/detectors/blitapp/blitapp_test.go +++ b/pkg/detectors/blitapp/blitapp_test.go @@ -39,7 +39,7 @@ func TestBlitApp_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"I_MncTA8nlFcqlBCakI5vwkwFD4_zRUYZKt8hyd"}, @@ -77,7 +77,7 @@ func TestBlitApp_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/blocknative/blocknative_test.go b/pkg/detectors/blocknative/blocknative_test.go index 406bacafc1d9..6b34453c1e20 100644 --- a/pkg/detectors/blocknative/blocknative_test.go +++ b/pkg/detectors/blocknative/blocknative_test.go @@ -39,7 +39,7 @@ func TestBlockNative_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"76e50995-059f-3d1a-af8e-cc85fc05eb03"}, @@ -77,7 +77,7 @@ func TestBlockNative_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/blogger/blogger.go b/pkg/detectors/blogger/blogger.go index 35516d862511..ab1274cef65d 100644 --- a/pkg/detectors/blogger/blogger.go +++ b/pkg/detectors/blogger/blogger.go @@ -22,8 +22,10 @@ var _ detectors.Detector = (*Scanner)(nil) var ( client = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blogger"}) + `\b([0-9A-Za-z-]{39})\b`) + // Google API keys always start with "AIzaSy" (6-char prefix + 33 remaining = 39 total). + // The prefix regex retains "blogger" proximity to distinguish from other Google API-backed things. + // No trailing \b because keys can end with '-', which is not a word character. + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blogger"}) + `\b(AIzaSy[0-9A-Za-z_-]{33})`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/blogger/blogger_test.go b/pkg/detectors/blogger/blogger_test.go index c7586c90f6e1..35419c71c45f 100644 --- a/pkg/detectors/blogger/blogger_test.go +++ b/pkg/detectors/blogger/blogger_test.go @@ -25,7 +25,7 @@ func TestBlogger_Pattern(t *testing.T) { input: ` func main() { // Create a new request with the secret as a header - req, err := http.NewRequest("GET", "https://api.example.com/v1/blogger/blogs?key=fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY", http.NoBody) + req, err := http.NewRequest("GET", "https://api.example.com/v1/blogger/blogs?key=AIzaSyDaGmWKa4JsXZ-HjGw7ISLn_3namBGewQe", http.NoBody) if err != nil { fmt.Println("Error creating request:", err) return @@ -34,7 +34,7 @@ func TestBlogger_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status if resp.StatusCode == http.StatusOK { @@ -44,28 +44,35 @@ func TestBlogger_Pattern(t *testing.T) { } } `, - want: []string{"fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY"}, + want: []string{"AIzaSyDaGmWKa4JsXZ-HjGw7ISLn_3namBGewQe"}, }, { name: "valid pattern - xml", input: ` - GLOBAL - {blogger} - {blogger AQAAABAAA mtkwpygpNROxOgLZCnEvl7gNme1IuFiQm9oxPzJ} - configuration for production + GLOBAL + {blogger} + {blogger AIzaSyCjkL9m2nOp4qRsTuVwXyZ0a1B2c3D4e5F} + configuration for production 2023-05-18T14:32:10Z - jenkins-admin + jenkins-admin `, - want: []string{"mtkwpygpNROxOgLZCnEvl7gNme1IuFiQm9oxPzJ"}, + want: []string{"AIzaSyCjkL9m2nOp4qRsTuVwXyZ0a1B2c3D4e5F"}, }, { - name: "invalid pattern", + name: "false positive - java class name (CSM-2238)", + input: ` + EJBLogger.logfinderReturnedMultipleValuesLoggable("@finderMethodName"); + `, + want: nil, + }, + { + name: "invalid pattern - wrong prefix", input: ` func main() { // Create a new request with the secret as a header - req, err := http.NewRequest("GET", "https://api.example.com/v1/blogger/blogs?key=fnWL(w7pz1t)6uCz-q6qocQZIxRF6S/UqePY", http.NoBody) + req, err := http.NewRequest("GET", "https://api.example.com/v1/blogger/blogs?key=fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY", http.NoBody) if err != nil { fmt.Println("Error creating request:", err) return @@ -74,7 +81,7 @@ func TestBlogger_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status if resp.StatusCode == http.StatusOK { @@ -86,6 +93,19 @@ func TestBlogger_Pattern(t *testing.T) { `, want: nil, }, + { + name: "invalid pattern - special characters", + input: ` + func main() { + req, err := http.NewRequest("GET", "https://api.example.com/v1/blogger/blogs?key=AIzaSy(w7pz1t)6uCz-q6qocQZIxRF6S/UqePY", http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + } + `, + want: nil, + }, } for _, test := range tests { diff --git a/pkg/detectors/boostnote/boostnote_test.go b/pkg/detectors/boostnote/boostnote_test.go index 871e2cabf674..223a85312b2a 100644 --- a/pkg/detectors/boostnote/boostnote_test.go +++ b/pkg/detectors/boostnote/boostnote_test.go @@ -39,7 +39,7 @@ func TestBoostNote_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"fb1026ac5994e3ad01799fe040289317ba2594a20e9e45307a143be82b49d213"}, @@ -77,7 +77,7 @@ func TestBoostNote_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/borgbase/borgbase_test.go b/pkg/detectors/borgbase/borgbase_test.go index 603979627532..628f54991862 100644 --- a/pkg/detectors/borgbase/borgbase_test.go +++ b/pkg/detectors/borgbase/borgbase_test.go @@ -40,7 +40,7 @@ func TestBorgBase_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"FoHclCFSi_aV09jowJQ4RUF_MiqW6ioqq6_OcyB0PFlV-mQ1yoFjk5JLlxbzRUzKTA6vsfR8wq6TNc83rtNKlkD092Sj1c9CbPVBXlHksy.sT2I/so6bMGdPcqxzbjrxYgAUiORgqJDeTet4gKOQlZpt"}, @@ -79,7 +79,7 @@ func TestBorgBase_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/boxoauth/boxoauth.go b/pkg/detectors/boxoauth/boxoauth.go index 75e7e51d31f4..5363e19e968f 100644 --- a/pkg/detectors/boxoauth/boxoauth.go +++ b/pkg/detectors/boxoauth/boxoauth.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "slices" "strings" regexp "github.com/wasilibs/go-re2" @@ -26,8 +27,19 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. clientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"id"}) + `\b([a-zA-Z0-9]{32})\b`) clientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"secret"}) + `\b([a-zA-Z0-9]{32})\b`) + // Box enterprise and user IDs are numeric strings. + subjectIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{ + "enterprise", "enterprise_id", "user", "user_id", "subject", "box_subject", + }) + `\b([0-9]{6,20})\b`) ) +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { @@ -52,34 +64,45 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result uniqueSecretMatches[match[1]] = struct{}{} } + var subjectIds []string + uniqueSubjectIdMatches := make(map[string]struct{}) + for _, match := range subjectIdPat.FindAllStringSubmatch(dataStr, -1) { + if _, seen := uniqueSubjectIdMatches[match[1]]; !seen { + uniqueSubjectIdMatches[match[1]] = struct{}{} + subjectIds = append(subjectIds, match[1]) + } + } + slices.Sort(subjectIds) + for resIdMatch := range uniqueIdMatches { for resSecretMatch := range uniqueSecretMatches { - - // ignore if the id and secret are the same if resIdMatch == resSecretMatch { continue } s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_BoxOauth, + DetectorType: s.Type(), Raw: []byte(resIdMatch), + RawV2: []byte(resIdMatch + resSecretMatch), SecretParts: map[string]string{ - "id": resIdMatch, - "secret": resSecretMatch, + "client_id": resIdMatch, + "client_secret": resSecretMatch, }, - RawV2: []byte(resIdMatch + resSecretMatch), } if verify { - client := s.client - if client == nil { - client = defaultClient - } - - isVerified, extraData, verificationErr := verifyMatch(ctx, client, resIdMatch, resSecretMatch) + isVerified, verificationErr := verifyMatch(ctx, s.getClient(), resIdMatch, resSecretMatch) s1.Verified = isVerified - s1.ExtraData = extraData s1.SetVerificationError(verificationErr, resIdMatch) + + if isVerified { + for _, subjectId := range subjectIds { + if verifySubjectID(ctx, s.getClient(), resIdMatch, resSecretMatch, subjectId) { + s1.SecretParts["subject_id"] = subjectId + break + } + } + } } results = append(results, s1) @@ -94,19 +117,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return } -func verifyMatch(ctx context.Context, client *http.Client, id string, secret string) (bool, map[string]string, error) { +func verifyMatch(ctx context.Context, client *http.Client, id string, secret string) (bool, error) { url := "https://api.box.com/oauth2/token" payload := strings.NewReader("grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, payload) if err != nil { - return false, nil, err + return false, err } req.Header = http.Header{"content-type": []string{"application/x-www-form-urlencoded"}} res, err := client.Do(req) if err != nil { - return false, nil, err + return false, err } defer func() { _, _ = io.Copy(io.Discard, res.Body) @@ -125,20 +148,53 @@ func verifyMatch(ctx context.Context, client *http.Client, id string, secret str { bodyBytes, err := io.ReadAll(res.Body) if err != nil { - return false, nil, err + return false, err } body := string(bodyBytes) if strings.Contains(body, "unauthorized_client") { - return true, nil, nil + return true, nil } else if strings.Contains(body, "invalid_client") { - return false, nil, nil + return false, nil } else { - return false, nil, fmt.Errorf("response body missing expected keyword") + return false, fmt.Errorf("response body missing expected keyword") } } default: - return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +// verifySubjectID checks whether a subject ID is valid for the given client +// credentials by attempting the Box CCG (Client Credentials Grant) auth flow. +// It mirrors the analyzer's Authenticate logic: try enterprise subject type +// first, then fall back to user subject type. Returns true on the first +// successful authentication (HTTP 200). +func verifySubjectID(ctx context.Context, client *http.Client, clientID, clientSecret, subjectID string) bool { + for _, subjectType := range []string{"enterprise", "user"} { + payload := fmt.Sprintf( + "grant_type=client_credentials&client_id=%s&client_secret=%s&box_subject_type=%s&box_subject_id=%s", + clientID, clientSecret, subjectType, subjectID, + ) + + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, "https://api.box.com/oauth2/token", strings.NewReader(payload)) + if err != nil { + continue + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + res, err := client.Do(req) + if err != nil { + continue + } + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + + if res.StatusCode == http.StatusOK { + return true + } } + return false } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/boxoauth/boxoauth_integration_test.go b/pkg/detectors/boxoauth/boxoauth_integration_test.go index 61a88b8e1913..a140d3408b34 100644 --- a/pkg/detectors/boxoauth/boxoauth_integration_test.go +++ b/pkg/detectors/boxoauth/boxoauth_integration_test.go @@ -54,6 +54,10 @@ func TestBoxOauth_FromChunk(t *testing.T) { Verified: true, Raw: []byte(id), RawV2: []byte(id + secret), + SecretParts: map[string]string{ + "client_id": id, + "client_secret": secret, + }, }, }, wantErr: false, @@ -72,6 +76,10 @@ func TestBoxOauth_FromChunk(t *testing.T) { Verified: false, Raw: []byte(id), RawV2: []byte(id + invalidSecret), + SecretParts: map[string]string{ + "client_id": id, + "client_secret": invalidSecret, + }, }, }, wantErr: false, diff --git a/pkg/detectors/boxoauth/boxoauth_test.go b/pkg/detectors/boxoauth/boxoauth_test.go index 473c609e3581..9573f98c377e 100644 --- a/pkg/detectors/boxoauth/boxoauth_test.go +++ b/pkg/detectors/boxoauth/boxoauth_test.go @@ -3,10 +3,12 @@ package boxoauth import ( "context" "fmt" + "net/http" "testing" - "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/h2non/gock.v1" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -17,6 +19,8 @@ var ( clientId = common.GenerateRandomPassword(true, true, true, false, 32) clientSecret = common.GenerateRandomPassword(true, true, true, false, 32) invalidClientSecret = common.GenerateRandomPassword(true, true, true, true, 32) + subjectId = "1234567890" + subjectId2 = "9876543210" ) func TestBoxOauth_Pattern(t *testing.T) { @@ -24,30 +28,58 @@ func TestBoxOauth_Pattern(t *testing.T) { ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) tests := []struct { - name string - input string - want []string + name string + input string + wantCount int + wantRawV2 string + wantMatched bool }{ { - name: "valid pattern", - input: fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, clientSecret), - want: []string{clientId + clientSecret}, + name: "valid pattern - no subject id", + input: fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, clientSecret), + wantCount: 1, + wantRawV2: clientId + clientSecret, + wantMatched: true, }, { - name: "invalid pattern", - input: fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, invalidClientSecret), - want: nil, + name: "valid pattern - with one subject id", + input: fmt.Sprintf("box id = '%s' box secret = '%s' enterprise = '%s'", clientId, clientSecret, subjectId), + wantCount: 1, + wantRawV2: clientId + clientSecret, + wantMatched: true, }, { - name: "invalid pattern", - input: fmt.Sprintf("box = '%s|%s'", clientId, invalidClientSecret), - want: nil, + name: "valid pattern - with multiple subject ids", + input: fmt.Sprintf("box id = '%s' box secret = '%s' enterprise = '%s' subject = '%s'", clientId, clientSecret, subjectId, subjectId2), + wantCount: 1, + wantRawV2: clientId + clientSecret, + wantMatched: true, + }, + { + name: "invalid pattern - secret contains special characters", + input: fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, invalidClientSecret), + wantCount: 0, + wantMatched: false, + }, + { + name: "invalid pattern - no keyword separation", + input: fmt.Sprintf("box = '%s|%s'", clientId, invalidClientSecret), + wantCount: 0, + wantMatched: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + + if !test.wantMatched { + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + assert.Empty(t, results) + return + } + if len(matchedDetectors) == 0 { t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords()) return @@ -56,28 +88,143 @@ func TestBoxOauth_Pattern(t *testing.T) { results, err := d.FromData(context.Background(), false, []byte(test.input)) require.NoError(t, err) - if len(results) != len(test.want) { - t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results)) - return - } - - actual := make(map[string]struct{}, len(results)) - for _, r := range results { - if len(r.RawV2) > 0 { - actual[string(r.RawV2)] = struct{}{} - } else { - actual[string(r.Raw)] = struct{}{} - } - } + require.Lenf(t, results, test.wantCount, + "expected %d results, got %d", test.wantCount, len(results)) - expected := make(map[string]struct{}, len(test.want)) - for _, v := range test.want { - expected[v] = struct{}{} - } - - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + for i, r := range results { + assert.Equalf(t, test.wantRawV2, string(r.RawV2), + "result[%d] RawV2 mismatch", i) } }) } } + +func TestBoxOauth_AnalysisInfo_VerifiedWithValidSubjectId(t *testing.T) { + client := common.SaneHttpClient() + d := Scanner{client: client} + + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + // Mock 1: verifyMatch → valid pair (400 with "unauthorized_client") + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusBadRequest). + BodyString(`{"error":"unauthorized_client","error_description":"The grant type is unauthorized for this client_id"}`) + + // Mock 2: verifySubjectID enterprise attempt → success (200) + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusOK). + BodyString(`{"access_token":"test_token","expires_in":4169,"token_type":"bearer"}`) + + input := fmt.Sprintf("box id = '%s' box secret = '%s' enterprise = '%s'", clientId, clientSecret, subjectId) + + results, err := d.FromData(context.Background(), true, []byte(input)) + require.NoError(t, err) + require.Len(t, results, 1) + + r := results[0] + assert.True(t, r.Verified) + require.NotNil(t, r.SecretParts) + assert.Equal(t, clientId, r.SecretParts["client_id"]) + assert.Equal(t, clientSecret, r.SecretParts["client_secret"]) + assert.Equal(t, subjectId, r.SecretParts["subject_id"]) +} + +func TestBoxOauth_AnalysisInfo_VerifiedNoSubjectId(t *testing.T) { + client := common.SaneHttpClient() + d := Scanner{client: client} + + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + // Mock: verifyMatch → valid pair + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusBadRequest). + BodyString(`{"error":"unauthorized_client","error_description":"The grant type is unauthorized for this client_id"}`) + + input := fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, clientSecret) + + results, err := d.FromData(context.Background(), true, []byte(input)) + require.NoError(t, err) + require.Len(t, results, 1) + + r := results[0] + assert.True(t, r.Verified) + require.NotNil(t, r.SecretParts) + assert.Equal(t, clientId, r.SecretParts["client_id"]) + assert.Equal(t, clientSecret, r.SecretParts["client_secret"]) + assert.Empty(t, r.SecretParts["subject_id"]) +} + +func TestBoxOauth_AnalysisInfo_VerifiedInvalidSubjectId(t *testing.T) { + client := common.SaneHttpClient() + d := Scanner{client: client} + + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + // Mock 1: verifyMatch → valid pair + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusBadRequest). + BodyString(`{"error":"unauthorized_client","error_description":"The grant type is unauthorized for this client_id"}`) + + // Mock 2: verifySubjectID enterprise attempt → fail (400) + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusBadRequest). + BodyString(`{"error":"invalid_grant","error_description":"Cannot obtain token based on the enterprise configuration for your app"}`) + + // Mock 3: verifySubjectID user attempt → fail (400) + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusBadRequest). + BodyString(`{"error":"invalid_grant","error_description":"Cannot obtain token based on the enterprise configuration for your app"}`) + + input := fmt.Sprintf("box id = '%s' box secret = '%s' enterprise = '%s'", clientId, clientSecret, subjectId) + + results, err := d.FromData(context.Background(), true, []byte(input)) + require.NoError(t, err) + require.Len(t, results, 1) + + r := results[0] + assert.True(t, r.Verified) + require.NotNil(t, r.SecretParts) + assert.Equal(t, clientId, r.SecretParts["client_id"]) + assert.Equal(t, clientSecret, r.SecretParts["client_secret"]) + assert.Empty(t, r.SecretParts["subject_id"]) +} + +func TestBoxOauth_AnalysisInfo_UnverifiedWithSubjectId(t *testing.T) { + client := common.SaneHttpClient() + d := Scanner{client: client} + + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + // Mock: verifyMatch → invalid pair (400 with "invalid_client") + gock.New("https://api.box.com"). + Post("/oauth2/token"). + Reply(http.StatusBadRequest). + BodyString(`{"error":"invalid_client","error_description":"The client_id is invalid"}`) + + input := fmt.Sprintf("box id = '%s' box secret = '%s' enterprise = '%s'", clientId, clientSecret, subjectId) + + results, err := d.FromData(context.Background(), true, []byte(input)) + require.NoError(t, err) + require.Len(t, results, 1) + + r := results[0] + assert.False(t, r.Verified) + require.NotNil(t, r.SecretParts) + assert.Equal(t, clientId, r.SecretParts["client_id"]) + assert.Equal(t, clientSecret, r.SecretParts["client_secret"]) + assert.Empty(t, r.SecretParts["subject_id"]) +} diff --git a/pkg/detectors/braintreepayments/braintreepayments.go b/pkg/detectors/braintreepayments/braintreepayments.go index 74578caec888..9704331bd9a8 100644 --- a/pkg/detectors/braintreepayments/braintreepayments.go +++ b/pkg/detectors/braintreepayments/braintreepayments.go @@ -113,7 +113,7 @@ func verifyBraintree(ctx context.Context, client *http.Client, url, pubKey, priv }() bodyString := string(bodyBytes) - if !(res.StatusCode == http.StatusOK) { + if res.StatusCode != http.StatusOK { return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } diff --git a/pkg/detectors/braintreepayments/braintreepayments_test.go b/pkg/detectors/braintreepayments/braintreepayments_test.go index 6c5ea244dfa5..281fc62187a9 100644 --- a/pkg/detectors/braintreepayments/braintreepayments_test.go +++ b/pkg/detectors/braintreepayments/braintreepayments_test.go @@ -40,7 +40,7 @@ func TestBrainTreePayments_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"f7b3cb83a7fdb915a71ce17ab8a903cc"}, @@ -79,7 +79,7 @@ func TestBrainTreePayments_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/braintrust/braintrust.go b/pkg/detectors/braintrust/braintrust.go new file mode 100644 index 000000000000..7293db4c2aea --- /dev/null +++ b/pkg/detectors/braintrust/braintrust.go @@ -0,0 +1,135 @@ +package braintrust + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // Braintrust API keys: + // Format: sk- + 48 alphanumeric characters (observed) + braintrustTokenPat = regexp.MustCompile( + `\b(sk-[A-Za-z0-9]{48})\b`, + ) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{"braintrust"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for Braintrust API tokens and optionally verifies them +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range braintrustTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_BrainTrustApiKey, + Raw: []byte(token), + Redacted: token[:8] + "...", + SecretParts: map[string]string{ + "token": token, + }, + } + + if verify { + verified, verificationErr := verifyBraintrustToken( + ctx, + s.getClient(), + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + + return +} + +func verifyBraintrustToken( + ctx context.Context, + client *http.Client, + token string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://api.braintrust.dev/v1/project?limit=1", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Bearer "+token) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + // Invalid or revoked token + return false, nil + case http.StatusForbidden: + // Valid token but lacks permission (still valid) + return true, nil + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_BrainTrustApiKey +} + +func (s Scanner) Description() string { + return "Braintrust is an AI evaluation and observability platform. Braintrust API keys can be used to access evaluation, observability, and AI development resources within the Braintrust platform." +} diff --git a/pkg/detectors/braintrust/braintrust_integration_test.go b/pkg/detectors/braintrust/braintrust_integration_test.go new file mode 100644 index 000000000000..72f7e288fc10 --- /dev/null +++ b/pkg/detectors/braintrust/braintrust_integration_test.go @@ -0,0 +1,181 @@ +//go:build detectors +// +build detectors + +package braintrust + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestBraintrust_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // Load secrets from GCP (same pattern as other detectors) + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeToken := testSecrets.MustGetField("BRAINTRUST_API_KEY") + inactiveToken := "sk-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Braintrust API key %s", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_BrainTrustApiKey, + Verified: true, + Raw: []byte(activeToken), + Redacted: activeToken[:8] + "...", + }, + }, + }, + { + name: "found, real token, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Braintrust API key %s", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_BrainTrustApiKey, + Verified: false, + Raw: []byte(activeToken), + Redacted: activeToken[:8] + "...", + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Braintrust API key %s", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_BrainTrustApiKey, + Verified: false, + Raw: []byte(activeToken), + Redacted: activeToken[:8] + "...", + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Braintrust API key %s", inactiveToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_BrainTrustApiKey, + Verified: false, + Raw: []byte(inactiveToken), + Redacted: inactiveToken[:8] + "...", + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("Braintrust.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Braintrust.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkBraintrust_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/braintrust/braintrust_test.go b/pkg/detectors/braintrust/braintrust_test.go new file mode 100644 index 000000000000..37f26b6db653 --- /dev/null +++ b/pkg/detectors/braintrust/braintrust_test.go @@ -0,0 +1,136 @@ +package braintrust + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestBraintrust_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - basic", + input: ` + [INFO] Starting AI eval + [DEBUG] braintrust_key=sk-JMwsKdC5JJfEGS4NhbQjzh1zfeYvGCuBkosFRWn98Z1H13Yg + `, + want: []string{ + "sk-JMwsKdC5JJfEGS4NhbQjzh1zfeYvGCuBkosFRWn98Z1H13Yg", + }, + }, + { + name: "valid pattern - with keyword nearby", + input: ` + [INFO] braintrust initialized + [DEBUG] BRAINTRUST_API_KEY=sk-76cnJ2Ns8wHZao70KdUdlZpBuSzej8gEokToNyeSPtd1RyZB + `, + want: []string{ + "sk-76cnJ2Ns8wHZao70KdUdlZpBuSzej8gEokToNyeSPtd1RyZB", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + braintrust_key1=sk-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + braintrust_key2=sk-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB + `, + want: []string{ + "sk-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "sk-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + }, + }, + { + name: "invalid pattern - too short", + input: ` + [DEBUG] braintrust_key=sk-1234 + `, + want: nil, + }, + { + name: "invalid pattern - too long", + input: ` + [DEBUG] braintrust_key=sk-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + `, + want: nil, + }, + { + name: "invalid pattern - invalid characters", + input: ` + [DEBUG] braintrust_key=sk-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA! + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + [INFO] braintrust API initialized sk- + `, + want: nil, + }, + { + name: "valid pattern - mixed alphanumeric", + input: ` + [DEBUG] braintrust_key=sk-a1B2c3D4e5F6g7H8i9J0kLmNoPqRsTuVwXyZ1234567890ao + `, + want: []string{ + "sk-a1B2c3D4e5F6g7H8i9J0kLmNoPqRsTuVwXyZ1234567890ao", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/brandfetch/v1/brandfetch_test.go b/pkg/detectors/brandfetch/v1/brandfetch_test.go index b06e6ea82586..f2846dd35a85 100644 --- a/pkg/detectors/brandfetch/v1/brandfetch_test.go +++ b/pkg/detectors/brandfetch/v1/brandfetch_test.go @@ -39,7 +39,7 @@ func TestBrandFetch_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"uHOAdwfQ7sD2yOpur72UqyUeIqnFwILOIlEPyBtJ"}, @@ -77,7 +77,7 @@ func TestBrandFetch_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/brandfetch/v2/brandfetch.go b/pkg/detectors/brandfetch/v2/brandfetch.go index 2dacb7ddd4d1..ed9ff4828634 100644 --- a/pkg/detectors/brandfetch/v2/brandfetch.go +++ b/pkg/detectors/brandfetch/v2/brandfetch.go @@ -95,7 +95,7 @@ func VerifyMatch(ctx context.Context, client *http.Client, token string) (bool, if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/brandfetch/v2/brandfetch_test.go b/pkg/detectors/brandfetch/v2/brandfetch_test.go index e991b6c929cb..b6c22fb3e54f 100644 --- a/pkg/detectors/brandfetch/v2/brandfetch_test.go +++ b/pkg/detectors/brandfetch/v2/brandfetch_test.go @@ -49,7 +49,7 @@ func TestBrandFetch_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status if resp.StatusCode == http.StatusOK { @@ -104,7 +104,7 @@ func TestBrandFetch_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/browserstack/browserstack.go b/pkg/detectors/browserstack/browserstack.go index 0804cbb630a0..1bae92601d08 100644 --- a/pkg/detectors/browserstack/browserstack.go +++ b/pkg/detectors/browserstack/browserstack.go @@ -106,7 +106,7 @@ func verifyBrowserStackCredentials(ctx context.Context, client *http.Client, use if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode == http.StatusOK { return true, nil diff --git a/pkg/detectors/browserstack/browserstack_test.go b/pkg/detectors/browserstack/browserstack_test.go index 97478c7509ea..2493bb8d0a28 100644 --- a/pkg/detectors/browserstack/browserstack_test.go +++ b/pkg/detectors/browserstack/browserstack_test.go @@ -44,7 +44,7 @@ func TestBrowserStack_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{ @@ -92,7 +92,7 @@ func TestBrowserStack_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/browshot/browshot_test.go b/pkg/detectors/browshot/browshot_test.go index 16ba249ba819..6fae5960d3a9 100644 --- a/pkg/detectors/browshot/browshot_test.go +++ b/pkg/detectors/browshot/browshot_test.go @@ -36,7 +36,7 @@ func TestBrowShot_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status if resp.StatusCode == http.StatusOK { @@ -78,7 +78,7 @@ func TestBrowShot_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bscscan/bscscan_test.go b/pkg/detectors/bscscan/bscscan_test.go index e5649eab933a..1614a61a419a 100644 --- a/pkg/detectors/bscscan/bscscan_test.go +++ b/pkg/detectors/bscscan/bscscan_test.go @@ -36,7 +36,7 @@ func TestBscScan_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"HYZHPP4PBYXCOZAVK4FH55W4MRHYLALPU1"}, @@ -71,7 +71,7 @@ func TestBscScan_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/buddyns/buddyns_test.go b/pkg/detectors/buddyns/buddyns_test.go index 078d9f7d6644..014df7627d2d 100644 --- a/pkg/detectors/buddyns/buddyns_test.go +++ b/pkg/detectors/buddyns/buddyns_test.go @@ -39,7 +39,7 @@ func TestBuddyNs_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"kkmvdiolccw4v0tue4lu7l7kmnnb4ao8z25ezink"}, @@ -77,7 +77,7 @@ func TestBuddyNs_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/budibase/budibase_test.go b/pkg/detectors/budibase/budibase_test.go index e5aa858a833c..130e757b5046 100644 --- a/pkg/detectors/budibase/budibase_test.go +++ b/pkg/detectors/budibase/budibase_test.go @@ -38,7 +38,7 @@ func TestBudiBase_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"b256def166fcdf4a429a1e83175105d5-fd36f3da1e934bf533cd0e68dbb80ed6a42e1178bd4200428d83e876e7d05e40b21e3a68888f826d"}, @@ -75,7 +75,7 @@ func TestBudiBase_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bugherd/bugherd_test.go b/pkg/detectors/bugherd/bugherd_test.go index c6088709adde..756a934cb41d 100644 --- a/pkg/detectors/bugherd/bugherd_test.go +++ b/pkg/detectors/bugherd/bugherd_test.go @@ -39,7 +39,7 @@ func TestBugHerd_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"fisy6bbu6il4x96bekx587"}, @@ -77,7 +77,7 @@ func TestBugHerd_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bugsnag/bugsnag_test.go b/pkg/detectors/bugsnag/bugsnag_test.go index 7b31971b201c..0b31beabca12 100644 --- a/pkg/detectors/bugsnag/bugsnag_test.go +++ b/pkg/detectors/bugsnag/bugsnag_test.go @@ -39,7 +39,7 @@ func TestBugSnag_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"wz9450iu-iewm-jonx-eab8-0ibxwadddm8i"}, @@ -77,7 +77,7 @@ func TestBugSnag_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/buildkite/v1/buildkite.go b/pkg/detectors/buildkite/v1/buildkite.go index b7ccfe09a22f..a77521562337 100644 --- a/pkg/detectors/buildkite/v1/buildkite.go +++ b/pkg/detectors/buildkite/v1/buildkite.go @@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_Buildkite, Raw: []byte(resMatch), ExtraData: make(map[string]string), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -60,12 +61,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.SetVerificationError(verificationErr, resMatch) s1.ExtraData = extraData - - if isVerified { - s1.SecretParts = map[string]string{ - "key": resMatch, - } - } } results = append(results, s1) diff --git a/pkg/detectors/buildkite/v1/buildkite_test.go b/pkg/detectors/buildkite/v1/buildkite_test.go index e4aa9d298040..7137736c9b7a 100644 --- a/pkg/detectors/buildkite/v1/buildkite_test.go +++ b/pkg/detectors/buildkite/v1/buildkite_test.go @@ -39,7 +39,7 @@ func TestBuildKite_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"kimu4axq3jxxdj8un0kpo3ua2ucr05zmhh4de0r6"}, @@ -77,7 +77,7 @@ func TestBuildKite_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/buildkite/v2/buildkite.go b/pkg/detectors/buildkite/v2/buildkite.go index 666e515e3c9d..93d77ee10e7a 100644 --- a/pkg/detectors/buildkite/v2/buildkite.go +++ b/pkg/detectors/buildkite/v2/buildkite.go @@ -45,6 +45,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Buildkite, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -52,13 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.SetVerificationError(verificationErr, resMatch) s1.ExtraData = extraData - - if isVerified { - s1.SecretParts = map[string]string{ - "key": resMatch, - } - } - } results = append(results, s1) diff --git a/pkg/detectors/buildkite/v2/buildkite_test.go b/pkg/detectors/buildkite/v2/buildkite_test.go index c7eeb19f6ca2..5f2b466a76f3 100644 --- a/pkg/detectors/buildkite/v2/buildkite_test.go +++ b/pkg/detectors/buildkite/v2/buildkite_test.go @@ -39,7 +39,7 @@ func TestBuildKiteV2_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"bkua_hqlh73m51jtho0jh12wcf2758c8fcdbv05z023ly"}, @@ -77,7 +77,7 @@ func TestBuildKiteV2_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bulbul/bulbul_test.go b/pkg/detectors/bulbul/bulbul_test.go index e9f4ced115f8..013ba9998e85 100644 --- a/pkg/detectors/bulbul/bulbul_test.go +++ b/pkg/detectors/bulbul/bulbul_test.go @@ -36,7 +36,7 @@ func TestBulBul_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"3kx19qpx748ldb75lsjicbs6ipit6ssm"}, @@ -71,7 +71,7 @@ func TestBulBul_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/bulksms/bulksms_test.go b/pkg/detectors/bulksms/bulksms_test.go index 792b8cfcc077..a33b5c2d5f7b 100644 --- a/pkg/detectors/bulksms/bulksms_test.go +++ b/pkg/detectors/bulksms/bulksms_test.go @@ -41,7 +41,7 @@ func TestBulkSMS_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"QGxPqRyzvt%xEKcV&#ePJGn)k0d9a381A26C47380B85F2DB572314-ACBDC267B-8"}, @@ -81,7 +81,7 @@ func TestBulkSMS_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/buttercms/buttercms_test.go b/pkg/detectors/buttercms/buttercms_test.go index 9fbd036921ac..b66c7f350c12 100644 --- a/pkg/detectors/buttercms/buttercms_test.go +++ b/pkg/detectors/buttercms/buttercms_test.go @@ -36,7 +36,7 @@ func TestButterCMS_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: []string{"l7psk7wkedkpiyp4jrx5fjdnno8c89243of6yde8"}, @@ -71,7 +71,7 @@ func TestButterCMS_Pattern(t *testing.T) { // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } `, want: nil, diff --git a/pkg/detectors/calendarific/calendarific.go b/pkg/detectors/calendarific/calendarific.go index 6aaa64cf2f00..7fbd77760a9c 100644 --- a/pkg/detectors/calendarific/calendarific.go +++ b/pkg/detectors/calendarific/calendarific.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/calendlyapikey/calendlyapikey.go b/pkg/detectors/calendlyapikey/calendlyapikey.go index 8323f930fcd3..5a4684b69f47 100644 --- a/pkg/detectors/calendlyapikey/calendlyapikey.go +++ b/pkg/detectors/calendlyapikey/calendlyapikey.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/calorieninja/calorieninja.go b/pkg/detectors/calorieninja/calorieninja.go index fd93a5689581..81c4f0201cd7 100644 --- a/pkg/detectors/calorieninja/calorieninja.go +++ b/pkg/detectors/calorieninja/calorieninja.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/campayn/campayn.go b/pkg/detectors/campayn/campayn.go index 9ea5e75b64da..8a4360eb11ef 100644 --- a/pkg/detectors/campayn/campayn.go +++ b/pkg/detectors/campayn/campayn.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", "TRUEREST apikey="+resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cannyio/cannyio.go b/pkg/detectors/cannyio/cannyio.go index 2bbcc490843b..aa74f9f95f9d 100644 --- a/pkg/detectors/cannyio/cannyio.go +++ b/pkg/detectors/cannyio/cannyio.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/capsulecrm/capsulecrm.go b/pkg/detectors/capsulecrm/capsulecrm.go index 645d35820f6e..faf21d95d02e 100644 --- a/pkg/detectors/capsulecrm/capsulecrm.go +++ b/pkg/detectors/capsulecrm/capsulecrm.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/captaindata/v1/captaindata.go b/pkg/detectors/captaindata/v1/captaindata.go index 55b0316a463f..43881832322b 100644 --- a/pkg/detectors/captaindata/v1/captaindata.go +++ b/pkg/detectors/captaindata/v1/captaindata.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/carboninterface/carboninterface.go b/pkg/detectors/carboninterface/carboninterface.go index 848b545917df..e02d76b092bf 100644 --- a/pkg/detectors/carboninterface/carboninterface.go +++ b/pkg/detectors/carboninterface/carboninterface.go @@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cashboard/cashboard.go b/pkg/detectors/cashboard/cashboard.go index 2fda6ef13904..8e7b08da1bc2 100644 --- a/pkg/detectors/cashboard/cashboard.go +++ b/pkg/detectors/cashboard/cashboard.go @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/caspio/caspio.go b/pkg/detectors/caspio/caspio.go index 3ae65f4d26be..137a490b3dc5 100644 --- a/pkg/detectors/caspio/caspio.go +++ b/pkg/detectors/caspio/caspio.go @@ -73,7 +73,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "text/plain") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/censys/censys.go b/pkg/detectors/censys/censys.go index 3db0f273c10f..3c311f2c6c32 100644 --- a/pkg/detectors/censys/censys.go +++ b/pkg/detectors/censys/censys.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/centralstationcrm/centralstationcrm.go b/pkg/detectors/centralstationcrm/centralstationcrm.go index 46127f3b715d..f5e22530e42c 100644 --- a/pkg/detectors/centralstationcrm/centralstationcrm.go +++ b/pkg/detectors/centralstationcrm/centralstationcrm.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-apikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cexio/cexio.go b/pkg/detectors/cexio/cexio.go index 1cb92a6108eb..47abeba4c72e 100644 --- a/pkg/detectors/cexio/cexio.go +++ b/pkg/detectors/cexio/cexio.go @@ -71,41 +71,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - - timestamp := strconv.FormatInt(time.Now().Unix()*1000, 10) - - signature := getCexIOPassphrase(resSecretMatch, resKeyMatch, timestamp, resUserIdMatch) - - payload := url.Values{} - payload.Add("key", resKeyMatch) - payload.Add("signature", signature) - payload.Add("nonce", timestamp) - - req, err := http.NewRequestWithContext(ctx, "POST", "https://cex.io/api/balance/", strings.NewReader(payload.Encode())) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(body) - validResponse := strings.Contains(bodyString, `timestamp`) - - var responseObject Response - if err := json.Unmarshal(body, &responseObject); err != nil { - continue - } - - if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resSecretMatch, resKeyMatch, resUserIdMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resSecretMatch, resKeyMatch, resUserIdMatch) } results = append(results, s1) @@ -116,6 +84,47 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resSecretMatch string, resKeyMatch string, resUserIdMatch string) (bool, error) { + + timestamp := strconv.FormatInt(time.Now().Unix()*1000, 10) + + signature := getCexIOPassphrase(resSecretMatch, resKeyMatch, timestamp, resUserIdMatch) + + payload := url.Values{} + payload.Add("key", resKeyMatch) + payload.Add("signature", signature) + payload.Add("nonce", timestamp) + + req, err := http.NewRequestWithContext(ctx, "POST", "https://cex.io/api/balance/", strings.NewReader(payload.Encode())) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(body) + validResponse := strings.Contains(bodyString, `timestamp`) + + var responseObject Response + if err := json.Unmarshal(body, &responseObject); err != nil { + return false, err + } + + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { + return true, nil + } + + return false, nil +} + type Response struct { Error string `json:"error"` } diff --git a/pkg/detectors/chartmogul/chartmogul.go b/pkg/detectors/chartmogul/chartmogul.go index d8f612a58689..a4a52ea05478 100644 --- a/pkg/detectors/chartmogul/chartmogul.go +++ b/pkg/detectors/chartmogul/chartmogul.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/chatbot/chatbot.go b/pkg/detectors/chatbot/chatbot.go index cb6b5f7a801a..60d54bd175ef 100644 --- a/pkg/detectors/chatbot/chatbot.go +++ b/pkg/detectors/chatbot/chatbot.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/chatfule/chatfule.go b/pkg/detectors/chatfule/chatfule.go index 6319dda2917b..91772f4ec0e1 100644 --- a/pkg/detectors/chatfule/chatfule.go +++ b/pkg/detectors/chatfule/chatfule.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/checio/checio.go b/pkg/detectors/checio/checio.go index 981e5abf29ce..587dc216b44a 100644 --- a/pkg/detectors/checio/checio.go +++ b/pkg/detectors/checio/checio.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/checklyhq/checklyhq.go b/pkg/detectors/checklyhq/checklyhq.go index a9d6c3bc81d9..6f81656467e2 100644 --- a/pkg/detectors/checklyhq/checklyhq.go +++ b/pkg/detectors/checklyhq/checklyhq.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/checkout/checkout.go b/pkg/detectors/checkout/checkout.go index 879005ca8d02..84f2234e8713 100644 --- a/pkg/detectors/checkout/checkout.go +++ b/pkg/detectors/checkout/checkout.go @@ -66,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/checkvist/checkvist.go b/pkg/detectors/checkvist/checkvist.go index 94f9260d0dd2..30c5b8731f23 100644 --- a/pkg/detectors/checkvist/checkvist.go +++ b/pkg/detectors/checkvist/checkvist.go @@ -71,7 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cicero/cicero.go b/pkg/detectors/cicero/cicero.go index 88271e94d049..801ee6dfac67 100644 --- a/pkg/detectors/cicero/cicero.go +++ b/pkg/detectors/cicero/cicero.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clarifai/clarifai.go b/pkg/detectors/clarifai/clarifai.go index a8feaf6c1c09..725c8da5292a 100644 --- a/pkg/detectors/clarifai/clarifai.go +++ b/pkg/detectors/clarifai/clarifai.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Key %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Key %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clearbit/clearbit.go b/pkg/detectors/clearbit/clearbit.go index 590cf028c0ae..e149a4fc1428 100644 --- a/pkg/detectors/clearbit/clearbit.go +++ b/pkg/detectors/clearbit/clearbit.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clicksendsms/clicksendsms.go b/pkg/detectors/clicksendsms/clicksendsms.go index 51e002c05e37..ef6a9b68a7d4 100644 --- a/pkg/detectors/clicksendsms/clicksendsms.go +++ b/pkg/detectors/clicksendsms/clicksendsms.go @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cliengo/cliengo.go b/pkg/detectors/cliengo/cliengo.go index 625f3d482274..c751b672c4be 100644 --- a/pkg/detectors/cliengo/cliengo.go +++ b/pkg/detectors/cliengo/cliengo.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Cliengo, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clinchpad/clinchpad.go b/pkg/detectors/clinchpad/clinchpad.go index 60ed04968a44..ab8f6d4d0f74 100644 --- a/pkg/detectors/clinchpad/clinchpad.go +++ b/pkg/detectors/clinchpad/clinchpad.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clockify/clockify.go b/pkg/detectors/clockify/clockify.go index b0e4f0e715a9..de4acfcac16e 100644 --- a/pkg/detectors/clockify/clockify.go +++ b/pkg/detectors/clockify/clockify.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Clockify, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clockworksms/clockworksms.go b/pkg/detectors/clockworksms/clockworksms.go index 4a167e5e7089..a3996e5fdb5b 100644 --- a/pkg/detectors/clockworksms/clockworksms.go +++ b/pkg/detectors/clockworksms/clockworksms.go @@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("access_token", tokenRes) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/closecrm/close.go b/pkg/detectors/closecrm/close.go index 41df5ec709af..aa7f579f55e5 100644 --- a/pkg/detectors/closecrm/close.go +++ b/pkg/detectors/closecrm/close.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloudconvert/cloudconvert.go b/pkg/detectors/cloudconvert/cloudconvert.go index 4d62b4fe2c70..0c2e48e93104 100644 --- a/pkg/detectors/cloudconvert/cloudconvert.go +++ b/pkg/detectors/cloudconvert/cloudconvert.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloudelements/cloudelements.go b/pkg/detectors/cloudelements/cloudelements.go index dfa788c86803..78e5f3b9357c 100644 --- a/pkg/detectors/cloudelements/cloudelements.go +++ b/pkg/detectors/cloudelements/cloudelements.go @@ -66,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("User %s, Organization %s", resMatch, resOrgMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloudflareapitoken/common.go b/pkg/detectors/cloudflareapitoken/common.go new file mode 100644 index 000000000000..18e50eed6485 --- /dev/null +++ b/pkg/detectors/cloudflareapitoken/common.go @@ -0,0 +1,68 @@ +package cloudflareapitoken + +import ( + "context" + "fmt" + "io" + "net/http" +) + +// VerifyUserToken checks if a Cloudflare user API token is valid. +// Returns (true, nil) if verified, (false, nil) for determinate auth +// failures, and (false, err) for indeterminate failures. +func VerifyUserToken(ctx context.Context, client *http.Client, token string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.cloudflare.com/client/v4/user/tokens/verify", nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} + +// VerifyAccountToken checks if a Cloudflare account API token is +// valid for the given account ID. Returns (true, nil) if verified, +// (false, nil) for determinate auth failures, and (false, err) for +// indeterminate failures. +func VerifyAccountToken(ctx context.Context, client *http.Client, token, accountID string) (bool, error) { + url := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/tokens/verify", accountID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go b/pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken.go similarity index 76% rename from pkg/detectors/cloudflareapitoken/cloudflareapitoken.go rename to pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken.go index 7645d81d7247..a6c72c0d1b7c 100644 --- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go +++ b/pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken.go @@ -2,21 +2,23 @@ package cloudflareapitoken import ( "context" - "fmt" - "net/http" "strings" regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + cfapitoken "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareapitoken" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) type Scanner struct{} -// Ensure the Scanner satisfies the interface at compile time. +// Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 1 } var ( client = common.SaneHttpClient() @@ -46,19 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.cloudflare.com/client/v4/user/tokens/verify", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + isVerified, verificationErr := cfapitoken.VerifyUserToken(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go b/pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken_integration_test.go similarity index 100% rename from pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go rename to pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken_integration_test.go diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go b/pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken_test.go similarity index 100% rename from pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go rename to pkg/detectors/cloudflareapitoken/v1/cloudflareapitoken_test.go diff --git a/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken.go b/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken.go new file mode 100644 index 000000000000..f79b15cc2fb1 --- /dev/null +++ b/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken.go @@ -0,0 +1,107 @@ +package cloudflareapitoken + +import ( + "context" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + cfapitoken "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareapitoken" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 2 } + +var ( + client = common.SaneHttpClient() + + // 2026+ formats: cfut_ (user token) and cfat_ (account token), self-identifying. + keyPat = regexp.MustCompile(`\b(cf[ua]t_[a-zA-Z0-9]{40}[a-f0-9]{8})\b`) + // Cloudflare account ID pattern for cfat_ token verification. + accountIDPat = regexp.MustCompile(`\b([a-f0-9]{32})\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { + return []string{"cfut_", "cfat_"} +} + +// FromData will find and optionally verify CloudflareApiToken secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + // Extract account IDs from surrounding data for cfat_ verification. + uniqueAccountIDs := make(map[string]struct{}) + for _, match := range accountIDPat.FindAllStringSubmatch(dataStr, -1) { + uniqueAccountIDs[match[1]] = struct{}{} + } + + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + if strings.HasPrefix(resMatch, "cfat_") { + // Account tokens: pair with each nearby account ID. + if len(uniqueAccountIDs) == 0 { + // No account ID found; still report the token. + results = append(results, detectors.Result{ + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, + }) + continue + } + for accountID := range uniqueAccountIDs { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Raw: []byte(resMatch), + RawV2: []byte(resMatch + accountID), + SecretParts: map[string]string{ + "key": resMatch, + "account_id": accountID, + }, + } + if verify { + isVerified, verificationErr := cfapitoken.VerifyAccountToken(ctx, client, resMatch, accountID) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + results = append(results, s1) + } + } else { + // cfut_ tokens use the user token verification endpoint. + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, + } + if verify { + isVerified, verificationErr := cfapitoken.VerifyUserToken(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + results = append(results, s1) + } + } + + return results, nil +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_CloudflareApiToken +} + +func (s Scanner) Description() string { + return "Cloudflare is a web infrastructure and website security company. Cloudflare API tokens (cfut_/cfat_ prefixed, 2026+ format) can be used to manage and interact with Cloudflare services." +} diff --git a/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken_integration_test.go b/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken_integration_test.go new file mode 100644 index 000000000000..39d8ee563839 --- /dev/null +++ b/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken_integration_test.go @@ -0,0 +1,157 @@ +//go:build detectors +// +build detectors + +package cloudflareapitoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestCloudflareApiTokenV2_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + userToken := testSecrets.MustGetField("CLOUDFLARE_USER_API_TOKEN_V2") + userTokenInactive := testSecrets.MustGetField("CLOUDFLARE_USER_API_TOKEN_V2_INACTIVE") + accountToken := testSecrets.MustGetField("CLOUDFLARE_ACCOUNT_API_TOKEN_V2") + accountID := testSecrets.MustGetField("CLOUDFLARE_ACCOUNT_API_TOKEN_V2_ACCOUNT_ID") + accountTokenInactive := testSecrets.MustGetField("CLOUDFLARE_ACCOUNT_API_TOKEN_V2_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "cfut_ found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("cloudflare token %s", userToken)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "cfut_ found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("cloudflare token %s", userTokenInactive)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "cfat_ found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("cloudflare token %s account %s", accountToken, accountID)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "cfat_ found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("cloudflare token %s account %s", accountTokenInactive, accountID)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_CloudflareApiToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("CloudflareApiTokenV2.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + got[i].RawV2 = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("CloudflareApiTokenV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken_test.go b/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken_test.go new file mode 100644 index 000000000000..395c688b47f5 --- /dev/null +++ b/pkg/detectors/cloudflareapitoken/v2/cloudflareapitoken_test.go @@ -0,0 +1,180 @@ +package cloudflareapitoken + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestCloudFlareAPITokenV2_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid v2 user token - go http client", + input: ` + func setupCloudflareClient() (*http.Client, error) { + req, err := http.NewRequest("GET", "https://api.cloudflare.com/client/v4/user", http.NoBody) + if err != nil { + return nil, err + } + // Rotated to the new self-identifying user token format during the March migration. + req.Header.Set("Authorization", "Bearer cfut_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfe") + + client := &http.Client{} + resp, _ := client.Do(req) + defer func() { _ = resp.Body.Close() }() + + return client, nil + } + `, + want: []string{ + "cfut_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfe", + }, + }, + { + name: "valid v2 account token - wrangler config", + input: ` + # wrangler.toml + name = "edge-worker" + main = "src/index.js" + compatibility_date = "2026-01-15" + + [env.production] + account_id = "a4c123b1612dd272d1371c17149d4395" + + # CF_API_TOKEN used by the deploy pipeline + CF_API_TOKEN = "cfat_OhbVrpoiVgRV5IfLBcbfnoGMbJmTPSIAoCLrZ3aW5da846a3" + `, + want: []string{ + "cfat_OhbVrpoiVgRV5IfLBcbfnoGMbJmTPSIAoCLrZ3aW5da846a3a4c123b1612dd272d1371c17149d4395", + }, + }, + { + name: "no match for legacy format", + input: ` + # .env.legacy + # Pre-2026 tokens don't carry the cfut_/cfat_ prefix and shouldn't match the new pattern. + CLOUDFLARE_API_TOKEN=kOjD1yceduu2jxL2uuwT9dkOIudU3_54sLCEud6j + `, + want: nil, + }, + { + name: "valid pattern - key rotation script with multiple tokens", + input: ` + #!/usr/bin/env bash + # rotate-cf-tokens.sh - retires the old worker token and provisions a replacement + set -euo pipefail + + echo "Retiring old token..." + OLD_TOKEN="cfut_fygw2wMqZcUDIh7yfJs1ON43xKmTecQoXsf2o3gy8eb5bb68" + + echo "Provisioning new token..." + NEW_TOKEN="cfut_S7RPeMOkIUpkDyr7OSJoRu1XXdo0cZuzren68K4Ta6fce484" + + curl -s -X DELETE "https://api.cloudflare.com/client/v4/user/tokens/verify" \ + -H "Authorization: Bearer ${OLD_TOKEN}" + curl -s -X PUT "https://api.cloudflare.com/client/v4/user/tokens/verify" \ + -H "Authorization: Bearer ${NEW_TOKEN}" + `, + want: []string{ + "cfut_fygw2wMqZcUDIh7yfJs1ON43xKmTecQoXsf2o3gy8eb5bb68", + "cfut_S7RPeMOkIUpkDyr7OSJoRu1XXdo0cZuzren68K4Ta6fce484", + }, + }, + { + name: "invalid pattern - too short", + input: ` + func setupCloudflareClient() { + // Truncated token accidentally committed during a config export + token := "cfut_ZE4CrcFhEIDXk9vL2sTLe" + client.SetToken(token) + } + `, + want: nil, + }, + { + name: "invalid pattern - too long", + input: ` + func setupCloudflareClient() { + // Extra characters appended by a bad find-and-replace + token := "cfut_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfeEXTRA" + client.SetToken(token) + } + `, + want: nil, + }, + { + name: "invalid pattern - invalid characters", + input: ` + func setupCloudflareClient() { + // Token was mangled when pasted from a Slack message with markdown formatting + token := "cfut_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbf!" + client.SetToken(token) + } + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + // TODO: load the real token instead of reading cfut_ from env + func isCloudflareUserToken(s string) bool { + return strings.HasPrefix(s, "cfut_") + } + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 && test.want != nil { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey.go b/pkg/detectors/cloudflarecakey/cloudflarecakey.go index 4b9bc5fef145..7d298acf6cb5 100644 --- a/pkg/detectors/cloudflarecakey/cloudflarecakey.go +++ b/pkg/detectors/cloudflarecakey/cloudflarecakey.go @@ -21,7 +21,10 @@ var _ detectors.Detector = (*Scanner)(nil) var ( client = common.SaneHttpClient() - // origin ca keys documentation: https://developers.cloudflare.com/fundamentals/api/get-started/ca-keys/ + // Origin CA keys (aka "Service Keys") are deprecated as of 2026-03-19: + // https://developers.cloudflare.com/changelog/post/2026-03-19-service-key-authentication-deprecated/ + // + // Reference: https://developers.cloudflare.com/fundamentals/api/get-started/ca-keys/ keyPat = regexp.MustCompile(`\b(v1\.0-[A-Za-z0-9-]{171})\b`) ) diff --git a/pkg/detectors/cloudflareglobalapikey/common.go b/pkg/detectors/cloudflareglobalapikey/common.go new file mode 100644 index 000000000000..73dc5a9fd21d --- /dev/null +++ b/pkg/detectors/cloudflareglobalapikey/common.go @@ -0,0 +1,39 @@ +package cloudflareglobalapikey + +import ( + "context" + "fmt" + "io" + "net/http" +) + +// VerifyGlobalAPIKey checks if a Cloudflare Global API Key is valid. +// Returns (true, nil) if verified, (false, nil) for determinate auth +// failures, and (false, err) for indeterminate failures. +func VerifyGlobalAPIKey(ctx context.Context, client *http.Client, apiKey, email string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.cloudflare.com/client/v4/user", nil) + if err != nil { + return false, err + } + req.Header.Add("X-Auth-Email", email) + req.Header.Add("X-Auth-Key", apiKey) + req.Header.Add("Content-Type", "application/json") + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go b/pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey.go similarity index 79% rename from pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go rename to pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey.go index ac2d268cdd4c..8337ba0b1934 100644 --- a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go +++ b/pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey.go @@ -2,13 +2,13 @@ package cloudflareglobalapikey import ( "context" - "net/http" "strings" regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + cfglobalapikey "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareglobalapikey" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -16,13 +16,17 @@ type Scanner struct { detectors.DefaultMultiPartCredentialProvider } -// Ensure the Scanner satisfies the interface at compile time. +// Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 1 } var ( client = common.SaneHttpClient() - apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudflare"}) + `\b([A-Za-z0-9_-]{37})\b`) + // Pre-2026 format: lowercase hex, 37-45 chars, requires "cloudflare" keyword nearby. + apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudflare"}) + `\b([a-f0-9]{37,45})\b`) emailPat = regexp.MustCompile(common.EmailPattern) ) @@ -60,21 +64,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.cloudflare.com/client/v4/user", nil) - if err != nil { - continue - } - req.Header.Add("X-Auth-Email", emailMatch) - req.Header.Add("X-Auth-Key", apiKeyRes) - req.Header.Add("Content-Type", "application/json") - - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + isVerified, verificationErr := cfglobalapikey.VerifyGlobalAPIKey(ctx, client, apiKeyRes, emailMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, apiKeyRes) } results = append(results, s1) diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_integration_test.go b/pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey_integration_test.go similarity index 100% rename from pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_integration_test.go rename to pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey_integration_test.go diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go b/pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey_test.go similarity index 88% rename from pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go rename to pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey_test.go index fef882a30b73..ab849aec66ef 100644 --- a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go +++ b/pkg/detectors/cloudflareglobalapikey/v1/cloudflareglobalapikey_test.go @@ -12,8 +12,8 @@ import ( ) var ( - validPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012 / testuser1005@example.com" - invalidPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012/testing@go" + validPattern = "abcdef1234567890abcdef1234567890abcdef0 / testuser1005@example.com" + invalidPattern = "abcdef1234567890abcdef1234567890abcdef0/testing@go" ) func TestCloudFlareGlobalAPIKey_Pattern(t *testing.T) { @@ -28,7 +28,7 @@ func TestCloudFlareGlobalAPIKey_Pattern(t *testing.T) { { name: "valid pattern", input: fmt.Sprintf("cloudflare: %s", validPattern), - want: []string{"abcD123efg456HIJklmn789OPQ_rstUVWxYZ-testuser1005@example.com"}, + want: []string{"abcdef1234567890abcdef1234567890abcdef0testuser1005@example.com"}, }, { name: "valid pattern - key out of prefix range", diff --git a/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey.go b/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey.go new file mode 100644 index 000000000000..3fac32b0d65c --- /dev/null +++ b/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey.go @@ -0,0 +1,94 @@ +package cloudflareglobalapikey + +import ( + "context" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + cfglobalapikey "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareglobalapikey" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 2 } + +var ( + client = common.SaneHttpClient() + + // 2026+ format: cfk_ prefix, sufficiently unique to match without keyword proximity. + apiKeyPat = regexp.MustCompile(`\b(cfk_[a-zA-Z0-9]{40}[a-f0-9]{8})\b`) + + emailPat = regexp.MustCompile(common.EmailPattern) +) + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { + return []string{"cfk_"} +} + +// FromData will find and optionally verify CloudflareGlobalApiKey secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + apiKeyMatches := apiKeyPat.FindAllStringSubmatch(dataStr, -1) + + uniqueEmailMatches := make(map[string]struct{}) + for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) { + uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{} + } + + for _, apiKeyMatch := range apiKeyMatches { + apiKeyRes := strings.TrimSpace(apiKeyMatch[1]) + + if len(uniqueEmailMatches) == 0 { + // No email found; still report the token unverified. + results = append(results, detectors.Result{ + DetectorType: detector_typepb.DetectorType_CloudflareGlobalApiKey, + Raw: []byte(apiKeyRes), + SecretParts: map[string]string{"key": apiKeyRes}, + }) + continue + } + + for emailMatch := range uniqueEmailMatches { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_CloudflareGlobalApiKey, + Redacted: emailMatch, + Raw: []byte(apiKeyRes), + RawV2: []byte(apiKeyRes + emailMatch), + SecretParts: map[string]string{ + "key": apiKeyRes, + "email": emailMatch, + }, + } + + if verify { + isVerified, verificationErr := cfglobalapikey.VerifyGlobalAPIKey(ctx, client, apiKeyRes, emailMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, apiKeyRes) + } + + results = append(results, s1) + } + } + + return results, nil +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_CloudflareGlobalApiKey +} + +func (s Scanner) Description() string { + return "Cloudflare is a web infrastructure and website security company. Cloudflare API keys (cfk_ prefixed, 2026+ format) can be used to access and modify these services." +} diff --git a/pkg/detectors/appoptics/appoptics_integration_test.go b/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey_integration_test.go similarity index 52% rename from pkg/detectors/appoptics/appoptics_integration_test.go rename to pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey_integration_test.go index db21dea664e5..3f18404d9b30 100644 --- a/pkg/detectors/appoptics/appoptics_integration_test.go +++ b/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey_integration_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package appoptics +package cloudflareglobalapikey import ( "context" @@ -9,24 +9,24 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - + "github.com/kylelemons/godebug/pretty" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -func TestAppoptics_FromChunk(t *testing.T) { +func TestCloudflareGlobalApiKeyV2_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("APPOPTICS") - inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE") + + globalApiKey := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_V2") + globalApiKeyEmail := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_V2_EMAIL") + inactiveGlobalApiKey := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_V2_INACTIVE") type args struct { ctx context.Context @@ -34,46 +34,45 @@ func TestAppoptics_FromChunk(t *testing.T) { verify bool } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + s Scanner + args args + want []detectors.Result + wantErr bool }{ { name: "found, verified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)), + data: []byte(fmt.Sprintf("cloudflare key %s with email %s", globalApiKey, globalApiKeyEmail)), verify: true, }, want: []detectors.Result{ { - DetectorType: detector_typepb.DetectorType_AppOptics, + DetectorType: detector_typepb.DetectorType_CloudflareGlobalApiKey, + Redacted: globalApiKeyEmail, Verified: true, }, }, - wantErr: false, - wantVerificationErr: false, + wantErr: false, }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("cloudflare key %s with email %s", inactiveGlobalApiKey, globalApiKeyEmail)), verify: true, }, want: []detectors.Result{ { - DetectorType: detector_typepb.DetectorType_AppOptics, + DetectorType: detector_typepb.DetectorType_CloudflareGlobalApiKey, + Redacted: globalApiKeyEmail, Verified: false, }, }, - wantErr: false, - wantVerificationErr: false, + wantErr: false, }, { name: "not found", @@ -83,29 +82,27 @@ func TestAppoptics_FromChunk(t *testing.T) { data: []byte("You cannot find the secret within"), verify: true, }, - want: nil, - wantErr: false, - wantVerificationErr: false, + want: nil, + wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { - t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("CloudflareGlobalApiKeyV2.FromData() error = %v, wantErr %v", err, tt.wantErr) return } for i := range got { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) - } + got[i].Raw = nil + got[i].RawV2 = nil } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("CloudflareGlobalApiKeyV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) } diff --git a/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey_test.go b/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey_test.go new file mode 100644 index 000000000000..9b031da4ee01 --- /dev/null +++ b/pkg/detectors/cloudflareglobalapikey/v2/cloudflareglobalapikey_test.go @@ -0,0 +1,163 @@ +package cloudflareglobalapikey + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestCloudFlareGlobalAPIKeyV2_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid v2 pattern - curl script with account email", + input: ` + #!/usr/bin/env bash + # purge-cache.sh - flushes the edge cache for the production zone + set -euo pipefail + + curl -s -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: testuser1005@example.com" \ + -H "X-Auth-Key: cfk_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfe" \ + -H "Content-Type: application/json" \ + --data '{"purge_everything":true}' + `, + want: []string{ + "cfk_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfetestuser1005@example.com", + }, + }, + { + name: "valid v2 pattern - env file, no email nearby still emits result", + input: ` + # .env.production + APP_NAME=edge-cache-worker + CLOUDFLARE_API_KEY=cfk_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfe + LOG_LEVEL=info + `, + want: []string{ + "cfk_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfe", + }, + }, + { + name: "no match for legacy format", + input: ` + # .env.legacy + # Pre-2026 global API keys are bare 37-char hex strings without a prefix. + CLOUDFLARE_API_KEY=abcdef1234567890abcdef1234567890abcdef0 + CLOUDFLARE_EMAIL=testuser1005@example.com + `, + want: nil, + }, + { + name: "valid pattern - ansible playbook rotating keys across environments", + input: ` + --- + - name: Rotate Cloudflare global API keys + hosts: localhost + vars: + staging_key: "cfk_fygw2wMqZcUDIh7yfJs1ON43xKmTecQoXsf2o3gy8eb5bb68" + production_key: "cfk_S7RPeMOkIUpkDyr7OSJoRu1XXdo0cZuzren68K4Ta6fce484" + tasks: + - name: Update staging secret store + command: "vault kv put secret/staging cf_key={{ staging_key }}" + - name: Update production secret store + command: "vault kv put secret/production cf_key={{ production_key }}" + `, + want: []string{ + "cfk_fygw2wMqZcUDIh7yfJs1ON43xKmTecQoXsf2o3gy8eb5bb68", + "cfk_S7RPeMOkIUpkDyr7OSJoRu1XXdo0cZuzren68K4Ta6fce484", + }, + }, + { + name: "invalid pattern - too short", + input: ` + func setupCloudflareClient() { + // Truncated key accidentally committed during a config export + key := "cfk_ZE4CrcFhEIDXk9vL2sTLe" + client.SetGlobalKey(key) + } + `, + want: nil, + }, + { + name: "invalid pattern - too long", + input: ` + func setupCloudflareClient() { + // Extra characters appended by a bad find-and-replace + key := "cfk_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbfeEXTRA" + client.SetGlobalKey(key) + } + `, + want: nil, + }, + { + name: "invalid pattern - invalid characters", + input: ` + func setupCloudflareClient() { + // Key was mangled when pasted from a Slack message with markdown formatting + key := "cfk_ZE4CrcFhEIDXk9vL2sTLeARsFp2ZZYbydVDhhIUq8573bbf!" + client.SetGlobalKey(key) + } + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + // TODO: load the real key instead of reading cfk_ from env + func isCloudflareGlobalKey(s string) bool { + return strings.HasPrefix(s, "cfk_") + } + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 && test.want != nil { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + t.Errorf("expected %d results, got %d", len(test.want), len(results)) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/cloudimage/cloudimage.go b/pkg/detectors/cloudimage/cloudimage.go index a992e427667d..2cf168664905 100644 --- a/pkg/detectors/cloudimage/cloudimage.go +++ b/pkg/detectors/cloudimage/cloudimage.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Client-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloudinary/cloudinary.go b/pkg/detectors/cloudinary/cloudinary.go new file mode 100644 index 000000000000..6db54f3afd84 --- /dev/null +++ b/pkg/detectors/cloudinary/cloudinary.go @@ -0,0 +1,134 @@ +package cloudinary + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // Cloudinary cloud names are typically b/w 3-50 characters. + // This regex matches cloud names that appear near the word "cloudinary" and either the keyword "name" or "@" (as in URLs). + cloudnamePat = regexp.MustCompile(`(?i:cloudinary)(?:.|[\n\r]){0,47}?` + `(?i:name|@)(?:.){0,10}?` + `\b([a-zA-Z][a-zA-Z0-9-]{2,49})\b`) + + // Cloudinary API keys are numeric and typically 15 digits long. + apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudinary"}) + `\b(\d{15})\b`) + + // Cloudinary API secrets are typically 27 characters long and may contain + // uppercase letters, lowercase letters, digits, underscores, or hyphens. + apiSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudinary"}) + `\b([A-Za-z0-9_-]{27})\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"cloudinary"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData will find and optionally verify DeepSeek secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueCloudNames := make(map[string]struct{}) + uniqueApiKeys := make(map[string]struct{}) + uniqueApiSecret := make(map[string]struct{}) + for _, match := range cloudnamePat.FindAllStringSubmatch(dataStr, -1) { + uniqueCloudNames[match[1]] = struct{}{} + } + for _, match := range apiKeyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueApiKeys[match[1]] = struct{}{} + } + for _, match := range apiSecretPat.FindAllStringSubmatch(dataStr, -1) { + uniqueApiSecret[match[1]] = struct{}{} + } + for cloudName := range uniqueCloudNames { + for apiKey := range uniqueApiKeys { + for apiSecret := range uniqueApiSecret { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_Cloudinary, + Raw: []byte(apiKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", cloudName, apiKey, apiSecret)), + SecretParts: map[string]string{ + "cloud_name": cloudName, + "api_key": apiKey, + "api_secret": apiSecret, + }, + } + if verify { + verified, verificationErr := verifyToken(ctx, s.getClient(), cloudName, apiKey, apiSecret) + s1.SetVerificationError(verificationErr, cloudName, apiKey, apiSecret) + s1.Verified = verified + } + results = append(results, s1) + } + } + } + return +} + +func verifyToken(ctx context.Context, client *http.Client, cloudName, apiKey, apiSecret string) (bool, error) { + u := &url.URL{ + Scheme: "https", + Host: "api.cloudinary.com", + Path: path.Join("v1_1", cloudName, "usage"), + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) + if err != nil { + return false, err + } + + req.SetBasicAuth(apiKey, apiSecret) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + // Invalid + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_Cloudinary +} + +func (s Scanner) Description() string { + return "Cloudinary is a cloud-based media management platform. Cloudinary API keys can be used to upload, manage, and deliver media assets." +} diff --git a/pkg/detectors/cloudinary/cloudinary_integration_test.go b/pkg/detectors/cloudinary/cloudinary_integration_test.go new file mode 100644 index 000000000000..04d2b0c72d2a --- /dev/null +++ b/pkg/detectors/cloudinary/cloudinary_integration_test.go @@ -0,0 +1,168 @@ +//go:build detectors +// +build detectors + +package cloudinary + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestCloudinary_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + cloudName := testSecrets.MustGetField("CLOUDINARY_CLOUD_NAME") + apiKey := testSecrets.MustGetField("CLOUDINARY_API_KEY") + apiSecret := testSecrets.MustGetField("CLOUDINARY_SECRET_KEY") + inactiveSecret := testSecrets.MustGetField("CLOUDINARY_INACTIVE_SECRET_KEY") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "You can find a Cloudinary apiSecret %s, Cloudinary apiKey %v and Cloudinary cloudName %v", apiSecret, apiKey, cloudName), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Cloudinary, + Verified: true, + Raw: []byte(apiKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", cloudName, apiKey, apiSecret)), + }, + }, + wantErr: false, + }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "You can find a Cloudinary apiSecret %s, Cloudinary apiKey %v and Cloudinary cloudName %v", apiSecret, apiKey, cloudName), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Cloudinary, + Verified: false, + Raw: []byte(apiKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", cloudName, apiKey, apiSecret)), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "You can find a Cloudinary apiSecret %s, Cloudinary apiKey %v and Cloudinary cloudName %v", apiSecret, apiKey, cloudName), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Cloudinary, + Verified: false, + Raw: []byte(apiKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", cloudName, apiKey, apiSecret)), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "You can find a Cloudinary inactiveapiSecret %s, Cloudinary apiKey %v and Cloudinary cloudName %v", inactiveSecret, apiKey, cloudName), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Cloudinary, + Verified: false, + Raw: []byte(apiKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", cloudName, apiKey, inactiveSecret)), + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Cloudinary.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "ExtraData", "verificationError", "primarySecret", "SecretParts") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Cloudinary.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/cloudinary/cloudinary_test.go b/pkg/detectors/cloudinary/cloudinary_test.go new file mode 100644 index 000000000000..7f3f2a828f7b --- /dev/null +++ b/pkg/detectors/cloudinary/cloudinary_test.go @@ -0,0 +1,151 @@ +package cloudinary + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestCloudinary_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: ` + [INFO] Sending request to the API + [DEBUG] Using cloudinary_apiSecret=ndiawudh1_wdajwoidjajawdeps + [DEBUG] Using cloudinary_apiKey=218873249723411 + [DEBUG] Using cloudinary_cloudName=wdjaiwojd + [INFO] Response received: 200 OK + `, + want: []string{"wdjaiwojd" + ":" + "218873249723411" + ":" + "ndiawudh1_wdajwoidjajawdeps"}, + }, + { + name: "valid pattern - cloudinary api environment variable", + input: ` + [INFO] Sending request to the cloudinary + [DEBUG] Using url=cloudinary://715268876851676:V-fwwRhcp3VrRPAqaFkLq3rpa60@fakwfpoaj + [INFO] Response received: 200 OK + `, + want: []string{"fakwfpoaj" + ":" + "715268876851676" + ":" + "V-fwwRhcp3VrRPAqaFkLq3rpa60"}, + }, + + { + name: "valid pattern - out of prefix range - apikey", + input: ` + [INFO] Sending request to the cloudinary + [DEBUG] Using cloudName=wdjaiwojd + [DEBUG] Using cloudinary_apiSecret=ndiawudh1_wdajwoidjajawdeps + [DEBUG] apiKey=218873249723411 + [INFO] Response received: 200 OK + `, + want: nil, + }, + { + name: "valid pattern - out of prefix range - apiSecret", + input: ` + [INFO] Sending request to the cloudinary + [DEBUG] Using cloudName=wdjaiwojd + [DEBUG] apiKey=218873249723411 + [DEBUG] Using apiSecret=ndiawudh1_wdajwoidjajawdeps + [INFO] Response received: 200 OK + `, + want: nil, + }, + { + name: "valid pattern - out of prefix range - cloudName", + input: ` + [INFO] Sending request to the cloudinary + [DEBUG] apiKey=218873249723411 + [DEBUG] Using apiSecret=ndiawudh1_wdajwoidjajawdeps + [INFO] Response received: 200 OK + [DEBUG] Used cloudName=wdjaiwojd + `, + want: nil, + }, + { + name: "valid pattern - only apikey", + input: ` + [INFO] Sending request to the cloudinary API + [DEBUG] Using apiKey=218873249723411 + `, + want: nil, + }, + { + name: "valid pattern - only secret", + input: ` + [INFO] Sending request to the cloudinary API + [DEBUG] Using apiSecret=ndiawudh1_wdajwoidjajawdeps + [INFO] Response received: 200 OK + `, + want: nil, + }, + { + name: "valid pattern - only cloudName", + input: ` + [INFO] Sending request to the cloudinary API + [DEBUG] Using cloudName=wdjaiwojd + [INFO] Response received: 200 OK + `, + want: nil, + }, + { + name: "invalid pattern - invalid api key length", + input: ` + [INFO] Sending request to the cloudinary API + [DEBUG] Using apikey=12312312432541444 + [DEBUG] Using apiSecret=ndiawudh1_wdajwoidjajawdeps + [DEBUG] Using cloudName=wdjaiwojd + [ERROR] Response received: 400 BadRequest + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords()) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results)) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/cloudmersive/cloudmersive.go b/pkg/detectors/cloudmersive/cloudmersive.go index 2589a73b16e5..a7c626c4b49d 100644 --- a/pkg/detectors/cloudmersive/cloudmersive.go +++ b/pkg/detectors/cloudmersive/cloudmersive.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Apikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloudplan/cloudplan.go b/pkg/detectors/cloudplan/cloudplan.go index cba92389be9a..817998bdfac1 100644 --- a/pkg/detectors/cloudplan/cloudplan.go +++ b/pkg/detectors/cloudplan/cloudplan.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("session_id", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloudsmith/cloudsmith.go b/pkg/detectors/cloudsmith/cloudsmith.go index a0278c16dd21..e400c7964b67 100644 --- a/pkg/detectors/cloudsmith/cloudsmith.go +++ b/pkg/detectors/cloudsmith/cloudsmith.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { var r response if err := json.NewDecoder(res.Body).Decode(&r); err != nil { diff --git a/pkg/detectors/cloverly/cloverly.go b/pkg/detectors/cloverly/cloverly.go index d9f22756436b..4a090286fc86 100644 --- a/pkg/detectors/cloverly/cloverly.go +++ b/pkg/detectors/cloverly/cloverly.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cloze/cloze.go b/pkg/detectors/cloze/cloze.go index eb22cd275780..ca821d8080b9 100644 --- a/pkg/detectors/cloze/cloze.go +++ b/pkg/detectors/cloze/cloze.go @@ -66,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/clustdoc/clustdoc.go b/pkg/detectors/clustdoc/clustdoc.go index fbe50ce134d5..9fcd611fe759 100644 --- a/pkg/detectors/clustdoc/clustdoc.go +++ b/pkg/detectors/clustdoc/clustdoc.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/coda/coda.go b/pkg/detectors/coda/coda.go index e89e51ab5e95..2d29c2571832 100644 --- a/pkg/detectors/coda/coda.go +++ b/pkg/detectors/coda/coda.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/codacy/codacy.go b/pkg/detectors/codacy/codacy.go index 3b6f5abf3830..562c155451ee 100644 --- a/pkg/detectors/codacy/codacy.go +++ b/pkg/detectors/codacy/codacy.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("api-token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/codeclimate/codeclimate.go b/pkg/detectors/codeclimate/codeclimate.go index fac992d69953..9875fd3b56d6 100644 --- a/pkg/detectors/codeclimate/codeclimate.go +++ b/pkg/detectors/codeclimate/codeclimate.go @@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { var r response if err := json.NewDecoder(res.Body).Decode(&r); err != nil { diff --git a/pkg/detectors/codemagic/codemagic.go b/pkg/detectors/codemagic/codemagic.go index 63f3afed9987..c22cd0591e63 100644 --- a/pkg/detectors/codemagic/codemagic.go +++ b/pkg/detectors/codemagic/codemagic.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-auth-token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/codequiry/codequiry.go b/pkg/detectors/codequiry/codequiry.go index d8d191ef2b62..7b3250f8b02d 100644 --- a/pkg/detectors/codequiry/codequiry.go +++ b/pkg/detectors/codequiry/codequiry.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("apikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/coinapi/coinapi.go b/pkg/detectors/coinapi/coinapi.go index 1732055a3db5..9d822b1c7618 100644 --- a/pkg/detectors/coinapi/coinapi.go +++ b/pkg/detectors/coinapi/coinapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-CoinAPI-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/coinbase/coinbase.go b/pkg/detectors/coinbase/coinbase.go index d446dd87c674..6b5b08c92698 100644 --- a/pkg/detectors/coinbase/coinbase.go +++ b/pkg/detectors/coinbase/coinbase.go @@ -106,18 +106,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_Coinbase, Raw: []byte(resPrivateKey), RawV2: []byte(fmt.Sprintf("%s:%s", resKeyName, resPrivateKey)), + SecretParts: map[string]string{ + "key_name": resKeyName, + "key": resPrivateKey, + }, } if verify { isVerified, verificationErr := s.verifyMatch(ctx, client, resKeyName, resPrivateKey) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resPrivateKey) - if isVerified { - s1.SecretParts = map[string]string{ - "key_name": resKeyName, - "key": resPrivateKey, - } - } } results = append(results, s1) diff --git a/pkg/detectors/coinlayer/coinlayer.go b/pkg/detectors/coinlayer/coinlayer.go index 71df5549c7cb..2fdcadf4bbbd 100644 --- a/pkg/detectors/coinlayer/coinlayer.go +++ b/pkg/detectors/coinlayer/coinlayer.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/coinlib/coinlib.go b/pkg/detectors/coinlib/coinlib.go index 8dfda739e1dd..a023bb75e9e6 100644 --- a/pkg/detectors/coinlib/coinlib.go +++ b/pkg/detectors/coinlib/coinlib.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/collect2/collect2.go b/pkg/detectors/collect2/collect2.go index 851142d7a1e1..753914bc009d 100644 --- a/pkg/detectors/collect2/collect2.go +++ b/pkg/detectors/collect2/collect2.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/column/column.go b/pkg/detectors/column/column.go index cf8173a422f8..a4e083a5c6d1 100644 --- a/pkg/detectors/column/column.go +++ b/pkg/detectors/column/column.go @@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(resMatch)))) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else { diff --git a/pkg/detectors/commercejs/commercejs.go b/pkg/detectors/commercejs/commercejs.go index 4c5209d91a12..82b00667318e 100644 --- a/pkg/detectors/commercejs/commercejs.go +++ b/pkg/detectors/commercejs/commercejs.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/commodities/commodities.go b/pkg/detectors/commodities/commodities.go index 6dc0e68844e4..2ae4c88f36de 100644 --- a/pkg/detectors/commodities/commodities.go +++ b/pkg/detectors/commodities/commodities.go @@ -48,28 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client.Timeout = 5 * time.Second - req, err := http.NewRequestWithContext(ctx, "GET", "https://commodities-api.com/api/latest?access_key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `"success":true`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -78,6 +59,35 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + client.Timeout = 5 * time.Second + req, err := http.NewRequestWithContext(ctx, "GET", "https://commodities-api.com/api/latest?access_key="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `"success":true`) + if res.StatusCode >= 200 && res.StatusCode < 300 { + if validResponse { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Commodities } diff --git a/pkg/detectors/companyhub/companyhub.go b/pkg/detectors/companyhub/companyhub.go index e6fca68f27f8..f1945be52b8c 100644 --- a/pkg/detectors/companyhub/companyhub.go +++ b/pkg/detectors/companyhub/companyhub.go @@ -66,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/confluent/confluent.go b/pkg/detectors/confluent/confluent.go index 530e3c4f3e6d..ae4157ef7914 100644 --- a/pkg/detectors/confluent/confluent.go +++ b/pkg/detectors/confluent/confluent.go @@ -68,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go index 30525dafb663..1c0884e89608 100644 --- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go +++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go @@ -41,6 +41,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_ContentfulPersonalAccessToken, Raw: []byte(keyRes), + Redacted: "..." + keyRes[len(keyRes)-4:], SecretParts: map[string]string{"key": keyRes}, } @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", keyRes)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go index 52c2b55b54a0..1ff0d4e6e72d 100644 --- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go +++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go @@ -2,6 +2,7 @@ package contentfulpersonalaccesstoken import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" @@ -10,6 +11,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) +const validToken = "CFPAT-fakeabcdefghijklmnopqrstuvwxyz0123456789xyz" + var ( validPattern = ` # Configuration File: config.yaml @@ -29,6 +32,12 @@ var ( # - Remember to rotate the secret every 90 days. # - The above credentials should only be used in a secure environment. ` + + validPatternWithToken = fmt.Sprintf(` + api: + auth_type: "Bearer" + contentful_access_token: "%s" + `, validToken) ) func TestContentfulPersonalAccessToken_Pattern(t *testing.T) { @@ -45,6 +54,11 @@ func TestContentfulPersonalAccessToken_Pattern(t *testing.T) { input: validPattern, want: nil, }, + { + name: "valid pattern with token - found", + input: validPatternWithToken, + want: []string{validToken}, + }, } for _, test := range tests { @@ -89,3 +103,22 @@ func TestContentfulPersonalAccessToken_Pattern(t *testing.T) { }) } } + +func TestContentfulPersonalAccessToken_SecretRedacted(t *testing.T) { + d := Scanner{} + + results, err := d.FromData(context.Background(), false, []byte(validPatternWithToken)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) == 0 { + t.Fatal("did not receive result") + } + + want := "..." + validToken[len(validToken)-4:] + if results[0].Redacted != want { + t.Errorf("expected redacted secret to be '%s', got '%s'", want, results[0].Redacted) + } +} diff --git a/pkg/detectors/conversiontools/conversiontools.go b/pkg/detectors/conversiontools/conversiontools.go index b0f9656c6025..d6393e18527a 100644 --- a/pkg/detectors/conversiontools/conversiontools.go +++ b/pkg/detectors/conversiontools/conversiontools.go @@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/convertapi/convertapi.go b/pkg/detectors/convertapi/convertapi.go index b70e2c90ecab..0430c171eae4 100644 --- a/pkg/detectors/convertapi/convertapi.go +++ b/pkg/detectors/convertapi/convertapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/convertkit/convertkit.go b/pkg/detectors/convertkit/convertkit.go index cc23ebfabae2..122baa87c004 100644 --- a/pkg/detectors/convertkit/convertkit.go +++ b/pkg/detectors/convertkit/convertkit.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/convier/convier.go b/pkg/detectors/convier/convier.go index ac890a525a55..a7154b5958fd 100644 --- a/pkg/detectors/convier/convier.go +++ b/pkg/detectors/convier/convier.go @@ -49,30 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - timeout := 10 * time.Second - client.Timeout = timeout - req, err := http.NewRequestWithContext(ctx, "POST", "https://convier.me/api/event", nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `"error":false`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -81,6 +60,37 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + timeout := 10 * time.Second + client.Timeout = timeout + req, err := http.NewRequestWithContext(ctx, "POST", "https://convier.me/api/event", nil) + if err != nil { + return false, err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `"error":false`) + if res.StatusCode >= 200 && res.StatusCode < 300 { + if validResponse { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Convier } diff --git a/pkg/detectors/countrylayer/countrylayer.go b/pkg/detectors/countrylayer/countrylayer.go index 245841d59682..400237f03b7f 100644 --- a/pkg/detectors/countrylayer/countrylayer.go +++ b/pkg/detectors/countrylayer/countrylayer.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/courier/courier.go b/pkg/detectors/courier/courier.go index 8ca8f52f7242..2afa8b9bd240 100644 --- a/pkg/detectors/courier/courier.go +++ b/pkg/detectors/courier/courier.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/coveralls/coveralls.go b/pkg/detectors/coveralls/coveralls.go index 2e6d4f1ef211..43f733040d4c 100644 --- a/pkg/detectors/coveralls/coveralls.go +++ b/pkg/detectors/coveralls/coveralls.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/craftmypdf/craftmypdf.go b/pkg/detectors/craftmypdf/craftmypdf.go index b4933bc891fa..afb26da9f4e1 100644 --- a/pkg/detectors/craftmypdf/craftmypdf.go +++ b/pkg/detectors/craftmypdf/craftmypdf.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-KEY", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/crowdin/crowdin.go b/pkg/detectors/crowdin/crowdin.go index 670f1770552b..8944bf6d7433 100644 --- a/pkg/detectors/crowdin/crowdin.go +++ b/pkg/detectors/crowdin/crowdin.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/cryptocompare/cryptocompare.go b/pkg/detectors/cryptocompare/cryptocompare.go index f8a5133dc67f..87563f1d065a 100644 --- a/pkg/detectors/cryptocompare/cryptocompare.go +++ b/pkg/detectors/cryptocompare/cryptocompare.go @@ -47,27 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://min-api.cryptocompare.com/data/blockchain/list?api_key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"Response":"Success"`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -76,6 +58,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://min-api.cryptocompare.com/data/blockchain/list?api_key="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(bodyBytes) + errCode := strings.Contains(bodyString, `"Response":"Success"`) + if res.StatusCode >= 200 && res.StatusCode < 300 { + if errCode { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_CryptoCompare } diff --git a/pkg/detectors/currencycloud/currencycloud.go b/pkg/detectors/currencycloud/currencycloud.go index 77e89fd424f6..9c3f90a59c7c 100644 --- a/pkg/detectors/currencycloud/currencycloud.go +++ b/pkg/detectors/currencycloud/currencycloud.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() bodyBytes, err := io.ReadAll(res.Body) if err != nil { continue diff --git a/pkg/detectors/currencyfreaks/currencyfreaks.go b/pkg/detectors/currencyfreaks/currencyfreaks.go index f7f782d6e042..c82272e2894c 100644 --- a/pkg/detectors/currencyfreaks/currencyfreaks.go +++ b/pkg/detectors/currencyfreaks/currencyfreaks.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/currencylayer/currencylayer.go b/pkg/detectors/currencylayer/currencylayer.go index d4e230783fce..ede0239182c1 100644 --- a/pkg/detectors/currencylayer/currencylayer.go +++ b/pkg/detectors/currencylayer/currencylayer.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err2 == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/currencyscoop/currencyscoop.go b/pkg/detectors/currencyscoop/currencyscoop.go index 2808a4c2db4b..a9c511a524c0 100644 --- a/pkg/detectors/currencyscoop/currencyscoop.go +++ b/pkg/detectors/currencyscoop/currencyscoop.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/customerguru/customerguru.go b/pkg/detectors/customerguru/customerguru.go index c235ea817e87..e82873834e88 100644 --- a/pkg/detectors/customerguru/customerguru.go +++ b/pkg/detectors/customerguru/customerguru.go @@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/customerio/customerio.go b/pkg/detectors/customerio/customerio.go index cc600eb0ec3f..c1dea10b1d63 100644 --- a/pkg/detectors/customerio/customerio.go +++ b/pkg/detectors/customerio/customerio.go @@ -70,7 +70,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/d7network/d7network.go b/pkg/detectors/d7network/d7network.go index 14b17d79b3f7..36f107de9d41 100644 --- a/pkg/detectors/d7network/d7network.go +++ b/pkg/detectors/d7network/d7network.go @@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", "Basic "+resMatch) res, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/dailyco/dailyco.go b/pkg/detectors/dailyco/dailyco.go index 1eae0ff5f3d4..0c77fcae5b6f 100644 --- a/pkg/detectors/dailyco/dailyco.go +++ b/pkg/detectors/dailyco/dailyco.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/dandelion/dandelion.go b/pkg/detectors/dandelion/dandelion.go index 6f1d83dc9da7..a498cafc60e5 100644 --- a/pkg/detectors/dandelion/dandelion.go +++ b/pkg/detectors/dandelion/dandelion.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/dareboost/dareboost.go b/pkg/detectors/dareboost/dareboost.go index 2dcc5128f89b..0ae3d7a2d8e7 100644 --- a/pkg/detectors/dareboost/dareboost.go +++ b/pkg/detectors/dareboost/dareboost.go @@ -47,32 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := strings.NewReader(`{ "token": "` + resMatch + `", "location": "Paris"}`) - - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.dareboost.com/0.8/config", payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `"status":200`) - - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -81,6 +58,39 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + payload := strings.NewReader(`{ "token": "` + resMatch + `", "location": "Paris"}`) + + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.dareboost.com/0.8/config", payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `"status":200`) + + if res.StatusCode >= 200 && res.StatusCode < 300 { + if validResponse { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Dareboost } diff --git a/pkg/detectors/databox/databox.go b/pkg/detectors/databox/databox.go index a2c4e2e3da60..5a11384dda41 100644 --- a/pkg/detectors/databox/databox.go +++ b/pkg/detectors/databox/databox.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Databox, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/databrickstoken/databrickstoken.go b/pkg/detectors/databrickstoken/databrickstoken.go index 82fdccb1706d..c6d124f0f10a 100644 --- a/pkg/detectors/databrickstoken/databrickstoken.go +++ b/pkg/detectors/databrickstoken/databrickstoken.go @@ -54,6 +54,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_DatabricksToken, Raw: []byte(token), RawV2: []byte(token + domain), + SecretParts: map[string]string{ + "token": token, + "domain": domain, + }, } if verify { @@ -65,13 +69,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyDatabricksToken(client, domain, token) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - - if s1.Verified { - s1.SecretParts = map[string]string{ - "token": token, - "domain": domain, - } - } } results = append(results, s1) diff --git a/pkg/detectors/datadogapikey/datadogapikey.go b/pkg/detectors/datadogapikey/datadogapikey.go index fca937d1b49a..b9eb3c404563 100644 --- a/pkg/detectors/datadogapikey/datadogapikey.go +++ b/pkg/detectors/datadogapikey/datadogapikey.go @@ -66,6 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_DatadogApikey, Raw: []byte(resApiMatch), + SecretParts: map[string]string{"api_key": resApiMatch}, } if verify { @@ -74,7 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyMatch(ctx, client, resApiMatch, baseURL) if isVerified { s1.Verified = isVerified - s1.SecretParts = map[string]string{"api_key": resApiMatch, "endpoint": baseURL} + s1.SecretParts["endpoint"] = baseURL // break the loop once we've successfully validated the token against a baseURL break } diff --git a/pkg/detectors/datadogapikey/datadogapikey_integration_test.go b/pkg/detectors/datadogapikey/datadogapikey_integration_test.go index d180f939517e..16d4403baf56 100644 --- a/pkg/detectors/datadogapikey/datadogapikey_integration_test.go +++ b/pkg/detectors/datadogapikey/datadogapikey_integration_test.go @@ -72,6 +72,9 @@ func TestDataDogApiKey_FromChunk(t *testing.T) { DetectorType: detector_typepb.DetectorType_DatadogApikey, Verified: false, Raw: []byte(invalidApiKey), + SecretParts: map[string]string{ + "api_key": invalidApiKey, + }, }, }, wantErr: false, diff --git a/pkg/detectors/datadogapikey/datadogapikey_test.go b/pkg/detectors/datadogapikey/datadogapikey_test.go index dca21a5268d4..a6495bcad514 100644 --- a/pkg/detectors/datadogapikey/datadogapikey_test.go +++ b/pkg/detectors/datadogapikey/datadogapikey_test.go @@ -27,6 +27,7 @@ func TestDataDogApiKey_Pattern_WithValidAPIKey(t *testing.T) { { DetectorType: detector_typepb.DetectorType_DatadogApikey, Raw: []byte(apiKey), + SecretParts: map[string]string{"api_key": apiKey}, }, } matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input)) @@ -40,7 +41,7 @@ func TestDataDogApiKey_Pattern_WithValidAPIKey(t *testing.T) { return } - if diff := cmp.Diff(wantedResult, results, cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "primarySecret")); diff != "" { + if diff := cmp.Diff(wantedResult, results, cmpopts.IgnoreUnexported(detectors.Result{})); diff != "" { t.Errorf("%s diff: (-want +got)\n%s", "TestDataDogApiKey_Pattern_WithValidAPIKeyOnly", diff) } } diff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.go index f5e9e68ea894..a3b07d726d78 100644 --- a/pkg/detectors/datadogtoken/datadogtoken.go +++ b/pkg/detectors/datadogtoken/datadogtoken.go @@ -3,6 +3,8 @@ package datadogtoken import ( "context" "encoding/json" + "fmt" + "io" "net/http" "strings" @@ -14,6 +16,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.EndpointSetter detectors.DefaultMultiPartCredentialProvider } @@ -26,7 +29,7 @@ var _ detectors.CloudProvider = (*Scanner)(nil) func (Scanner) CloudEndpoint() string { return "https://api.datadoghq.com" } var ( - client = common.SaneHttpClient() + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`) @@ -93,6 +96,13 @@ func setOrganizationInfo(opt []*options, s1 *detectors.Result) { } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { @@ -114,6 +124,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for endpoint := range uniqueFoundUrls { endpoints = append(endpoints, endpoint) } + client := s.getClient() for _, apiMatch := range apiMatches { resApiMatch := strings.TrimSpace(apiMatch[1]) @@ -126,44 +137,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result ExtraData: map[string]string{ "Type": "Application+APIKey", }, + SecretParts: map[string]string{"api_key": resApiMatch, "app_key": resAppMatch}, } if verify { for _, baseURL := range s.Endpoints(endpoints...) { - req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("DD-API-KEY", resApiMatch) - req.Header.Add("DD-APPLICATION-KEY", resAppMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - s1.SecretParts = map[string]string{"api_key": resApiMatch, "app_key": resAppMatch, "endpoint": baseURL} - var serviceResponse userServiceResponse - if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err == nil { - // setup emails - if len(serviceResponse.Data) > 0 { - setUserEmails(serviceResponse.Data, &s1) - } - // setup organizations - if len(serviceResponse.Included) > 0 { - setOrganizationInfo(serviceResponse.Included, &s1) - } - } - // break the loop once we've successfully validated the token against a baseURL - break + res, isVerified, verificationErr := verifyMatch(ctx, client, resApiMatch, resAppMatch, baseURL) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resApiMatch, resAppMatch) + s1.SecretParts["endpoint"] = baseURL + if isVerified && res != nil { + if len(res.Data) > 0 { + setUserEmails(res.Data, &s1) } + if len(res.Included) > 0 { + setOrganizationInfo(res.Included, &s1) + } + break } } } results = append(results, s1) } } - return results, nil } @@ -174,3 +170,38 @@ func (s Scanner) Type() detector_typepb.DetectorType { func (s Scanner) Description() string { return "Datadog is a monitoring and security platform for cloud applications. Datadog API and Application keys can be used to access and manage data and configurations within Datadog." } + +func verifyMatch(ctx context.Context, client *http.Client, apiKey, appKey, baseUrl string) (*userServiceResponse, bool, error) { + // Reference: https://docs.datadoghq.com/api/latest/users/ + + req, err := http.NewRequestWithContext(ctx, "GET", baseUrl+"/api/v2/users", nil) + if err != nil { + return nil, false, err + } + + req.Header.Add("Content-Type", "application/json") + req.Header.Add("DD-API-KEY", apiKey) + req.Header.Add("DD-APPLICATION-KEY", appKey) + res, err := client.Do(req) + if err != nil { + return nil, false, err + } + + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + var serviceResponse userServiceResponse + if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err != nil { + return nil, true, nil + } + return &serviceResponse, true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return nil, false, nil + default: + return nil, false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/datadogtoken/datadogtoken_integration_test.go b/pkg/detectors/datadogtoken/datadogtoken_integration_test.go index e33450919cb0..ca829394d085 100644 --- a/pkg/detectors/datadogtoken/datadogtoken_integration_test.go +++ b/pkg/detectors/datadogtoken/datadogtoken_integration_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -79,6 +80,11 @@ func TestDatadogToken_FromChunk(t *testing.T) { ExtraData: map[string]string{ "Type": "Application+APIKey", }, + SecretParts: map[string]string{ + "api_key": apiKey, + "app_key": inactiveAppKey, + "endpoint": "https://api.datadoghq.com", // default endpoint + }, }, }, wantErr: false, @@ -117,13 +123,72 @@ func TestDatadogToken_FromChunk(t *testing.T) { got[i].RawV2 = nil delete(got[i].ExtraData, "user_emails") } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "chunkOffset", + "chunkOffsetSet", + ) + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("DatadogToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) } } +func TestDatadogToken_FromChunk_Unverified(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN") + inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE") + + data := []byte(fmt.Sprintf( + "You can find a datadogtoken secret %s within but datadog %s not valid", + inactiveAppKey, + apiKey, + )) + + s := Scanner{} + s.UseCloudEndpoint(true) + s.SetCloudEndpoint(s.CloudEndpoint()) + s.UseFoundEndpoints(true) + + results, err := s.FromData(ctx, true, data) + if err != nil { + t.Fatalf("FromData returned error: %v", err) + } + + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + r := results[0] + + if r.DetectorType != detector_typepb.DetectorType_DatadogToken { + t.Errorf("unexpected detector type: %v", r.DetectorType) + } + + if r.Verified { + t.Errorf("expected token to be unverified") + } + + if got := r.ExtraData["Type"]; got != "Application+APIKey" { + t.Errorf("unexpected ExtraData Type: %q", got) + } + + if len(r.Raw) == 0 { + t.Errorf("expected raw secret to be present") + } +} + func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := Scanner{} diff --git a/pkg/detectors/datagov/datagov.go b/pkg/detectors/datagov/datagov.go index a5cb91f89393..8fe15d5ad20a 100644 --- a/pkg/detectors/datagov/datagov.go +++ b/pkg/detectors/datagov/datagov.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/debounce/debounce.go b/pkg/detectors/debounce/debounce.go index a2a535020447..12781f3e52e7 100644 --- a/pkg/detectors/debounce/debounce.go +++ b/pkg/detectors/debounce/debounce.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/deepai/deepai.go b/pkg/detectors/deepai/deepai.go index 686bd07bffef..befc7e70171a 100644 --- a/pkg/detectors/deepai/deepai.go +++ b/pkg/detectors/deepai/deepai.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - writer.Close() + _ = writer.Close() req, err := http.NewRequestWithContext(ctx, "POST", "https://api.deepai.org/api/text-tagging", bytes.NewReader(body.Bytes())) if err != nil { continue @@ -68,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/deepgram/deepgram.go b/pkg/detectors/deepgram/deepgram.go index 18e41438ed7e..92991cf73a67 100644 --- a/pkg/detectors/deepgram/deepgram.go +++ b/pkg/detectors/deepgram/deepgram.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/delighted/delighted.go b/pkg/detectors/delighted/delighted.go index fe8f42998709..748ec5e813ae 100644 --- a/pkg/detectors/delighted/delighted.go +++ b/pkg/detectors/delighted/delighted.go @@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/demio/demio.go b/pkg/detectors/demio/demio.go index a94921fc83e4..b1ef8ffe72f5 100644 --- a/pkg/detectors/demio/demio.go +++ b/pkg/detectors/demio/demio.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/deno/denodeploy.go b/pkg/detectors/deno/denodeploy.go index 838fd3942af9..c0cc4a822a29 100644 --- a/pkg/detectors/deno/denodeploy.go +++ b/pkg/detectors/deno/denodeploy.go @@ -65,8 +65,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() - if res.StatusCode == 200 { + defer func() { _ = res.Body.Close() }() + switch res.StatusCode { + case 200: s1.Verified = true body, err := io.ReadAll(res.Body) @@ -82,9 +83,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } } } - } else if res.StatusCode == 401 { + case 401: // The secret is determinately not verified (nothing to do) - } else { + default: err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) s1.SetVerificationError(err, token) } diff --git a/pkg/detectors/deputy/deputy.go b/pkg/detectors/deputy/deputy.go index 10e1fa4f9c40..1d27b4a122fc 100644 --- a/pkg/detectors/deputy/deputy.go +++ b/pkg/detectors/deputy/deputy.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("OAuth %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/detectify/detectify.go b/pkg/detectors/detectify/detectify.go index 82484c723a9f..ba9c025300f6 100644 --- a/pkg/detectors/detectify/detectify.go +++ b/pkg/detectors/detectify/detectify.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Detectify-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/detectlanguage/detectlanguage.go b/pkg/detectors/detectlanguage/detectlanguage.go index 55fefb6de6a5..10a04f91e580 100644 --- a/pkg/detectors/detectlanguage/detectlanguage.go +++ b/pkg/detectors/detectlanguage/detectlanguage.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index fc781af1d6a5..dbcec64f265f 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -49,7 +49,7 @@ type Detector interface { type CustomResultsCleaner interface { // CleanResults removes "superfluous" results from a result set (where the definition of "superfluous" is detector- // specific). - CleanResults(results []Result) []Result + CleanResults(results []Result, verificationEnabled bool) []Result // ShouldCleanResultsIrrespectiveOfConfiguration allows a custom cleaner to instruct the engine to ignore // user-provided configuration that controls whether results are cleaned. (User-provided configuration is not the // only factor that determines whether the engine runs cleaning logic.) @@ -134,6 +134,11 @@ type Result struct { Value string Line int64 } + + // chunkOffset stores the byte position of this result's secret within chunk data. + // Used to disambiguate line numbers when the same secret appears multiple times. + chunkOffset int64 + chunkOffsetSet bool } // CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when @@ -176,6 +181,33 @@ func (r *Result) GetPrimarySecretValue() string { return r.primarySecret.Value } +// ClearSecrets removes fields that may contain raw secret material. +func (r *Result) ClearSecrets() { + r.Raw = nil + r.RawV2 = nil + r.SecretParts = nil + r.primarySecret = struct { + Value string + Line int64 + }{} +} + +// SetChunkOffset records the byte position of this result's secret within the chunk data. +func (r *Result) SetChunkOffset(offset int64) { + r.chunkOffset = offset + r.chunkOffsetSet = true +} + +// ChunkOffset returns the byte position of this result's secret within the chunk data. +func (r *Result) ChunkOffset() int64 { + return r.chunkOffset +} + +// HasChunkOffset reports whether a chunk offset has been explicitly set on this result. +func (r *Result) HasChunkOffset() bool { + return r.chunkOffsetSet +} + // redactSecrets replaces all instances of the given secrets with [REDACTED] in the error message. func redactSecrets(err error, secrets ...string) error { lastErr := unwrapToLast(err) @@ -248,7 +280,7 @@ func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata { // CleanResults returns all verified secrets, and if there are no verified secrets, // just one unverified secret if there are any. -func CleanResults(results []Result) []Result { +func CleanResults(results []Result, _ bool) []Result { if len(results) == 0 { return results } @@ -354,3 +386,10 @@ func withDedupKey(ctx context.Context, detType detector_typepb.DetectorType, cre func DoWithDedup(client *http.Client, detType detector_typepb.DetectorType, credential string, req *http.Request) (*http.Response, error) { return client.Do(req.WithContext(withDedupKey(req.Context(), detType, credential))) } + +// ResultVerifier is an optional interface that a detector can implement to verify a single +// previously-extracted result independently of the chunk it came from, which lets the +// verification cache verify only cache misses instead of re-verifying an entire chunk. +type ResultVerifier interface { + VerifyResult(ctx context.Context, result *Result) +} diff --git a/pkg/detectors/dfuse/dfuse.go b/pkg/detectors/dfuse/dfuse.go index c57bcc3f9923..408b35fb849d 100644 --- a/pkg/detectors/dfuse/dfuse.go +++ b/pkg/detectors/dfuse/dfuse.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/diffbot/diffbot.go b/pkg/detectors/diffbot/diffbot.go index 0b5385a08be2..63f77d4a6dcc 100644 --- a/pkg/detectors/diffbot/diffbot.go +++ b/pkg/detectors/diffbot/diffbot.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"token":`) && strings.Contains(bodyString, `"planCredits":`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/diggernaut/diggernaut.go b/pkg/detectors/diggernaut/diggernaut.go index 13754ce1cbc9..4d28aadad84f 100644 --- a/pkg/detectors/diggernaut/diggernaut.go +++ b/pkg/detectors/diggernaut/diggernaut.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/digitaloceantoken/digitaloceantoken.go b/pkg/detectors/digitaloceantoken/digitaloceantoken.go index e60fd3df8d22..43bace232b50 100644 --- a/pkg/detectors/digitaloceantoken/digitaloceantoken.go +++ b/pkg/detectors/digitaloceantoken/digitaloceantoken.go @@ -43,6 +43,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_DigitalOceanToken, Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } if verify { @@ -53,11 +54,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyDigitalOceanToken(ctx, client, token) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": token, - } - } } results = append(results, s1) @@ -79,7 +75,7 @@ func verifyDigitalOceanToken(ctx context.Context, client *http.Client, token str if err != nil { return false, fmt.Errorf("failed to make request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/digitaloceanv2/digitaloceanv2.go b/pkg/detectors/digitaloceanv2/digitaloceanv2.go index b42117fe23aa..215c4190256c 100644 --- a/pkg/detectors/digitaloceanv2/digitaloceanv2.go +++ b/pkg/detectors/digitaloceanv2/digitaloceanv2.go @@ -49,6 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_DigitalOceanV2, Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } if verify { @@ -60,23 +61,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // Check if the token is a refresh token or an access token switch { case strings.HasPrefix(token, "dor_v1_"): - verified, verificationErr, newAccessToken := verifyRefreshToken(ctx, client, token) + verified, newAccessToken, verificationErr := verifyRefreshToken(ctx, client, token) s1.SetVerificationError(verificationErr) s1.Verified = verified if s1.Verified { - s1.SecretParts = map[string]string{ - "key": newAccessToken, - } + s1.SecretParts["key"] = newAccessToken } case strings.HasPrefix(token, "doo_v1_"), strings.HasPrefix(token, "dop_v1_"): verified, verificationErr := verifyAccessToken(ctx, client, token) s1.Verified = verified s1.SetVerificationError(verificationErr) - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": token, - } - } } } @@ -90,42 +84,42 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // If the token is valid, it returns the new access token and no error. // If the token is invalid/expired, it returns an empty string and no error. // If an error is encountered, it returns an empty string along and the error. -func verifyRefreshToken(ctx context.Context, client *http.Client, token string) (bool, error, string) { +func verifyRefreshToken(ctx context.Context, client *http.Client, token string) (bool, string, error) { // Ref: https://docs.digitalocean.com/reference/api/oauth/ url := "https://cloud.digitalocean.com/v1/oauth/token?grant_type=refresh_token&refresh_token=" + token req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { - return false, fmt.Errorf("failed to create request: %w", err), "" + return false, "", fmt.Errorf("failed to create request: %w", err) } res, err := client.Do(req) if err != nil { - return false, fmt.Errorf("failed to make request: %w", err), "" + return false, "", fmt.Errorf("failed to make request: %w", err) } bodyBytes, err := io.ReadAll(res.Body) if err != nil { - return false, fmt.Errorf("failed to read response body: %w", err), "" + return false, "", fmt.Errorf("failed to read response body: %w", err) } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: var responseMap map[string]interface{} if err := json.Unmarshal(bodyBytes, &responseMap); err != nil { - return false, fmt.Errorf("failed to parse response body: %w", err), "" + return false, "", fmt.Errorf("failed to parse response body: %w", err) } // Extract the access token from the response accessToken, exists := responseMap["access_token"].(string) if !exists { - return false, fmt.Errorf("access_token not found in response: %s", string(bodyBytes)), "" + return false, "", fmt.Errorf("access_token not found in response: %s", string(bodyBytes)) } - return true, nil, accessToken + return true, accessToken, nil case http.StatusUnauthorized: - return false, nil, "" + return false, "", nil default: - return false, fmt.Errorf("unexpected status code: %d", res.StatusCode), "" + return false, "", fmt.Errorf("unexpected status code: %d", res.StatusCode) } } @@ -147,7 +141,7 @@ func verifyAccessToken(ctx context.Context, client *http.Client, token string) ( if err != nil { return false, fmt.Errorf("failed to make request: %w", err) } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/discordbottoken/discordbottoken.go b/pkg/detectors/discordbottoken/discordbottoken.go index e1b0d33600bc..432b8dc8e6ef 100644 --- a/pkg/detectors/discordbottoken/discordbottoken.go +++ b/pkg/detectors/discordbottoken/discordbottoken.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bot %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/discordwebhook/discordwebhook.go b/pkg/detectors/discordwebhook/discordwebhook.go index 6fb040c048a3..97e177543f98 100644 --- a/pkg/detectors/discordwebhook/discordwebhook.go +++ b/pkg/detectors/discordwebhook/discordwebhook.go @@ -48,7 +48,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/disqus/disqus.go b/pkg/detectors/disqus/disqus.go index 119bfd1c358d..7d304b1cf743 100644 --- a/pkg/detectors/disqus/disqus.go +++ b/pkg/detectors/disqus/disqus.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ditto/ditto.go b/pkg/detectors/ditto/ditto.go index 1655eaee6f41..47e589fa2450 100644 --- a/pkg/detectors/ditto/ditto.go +++ b/pkg/detectors/ditto/ditto.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/dnscheck/dnscheck.go b/pkg/detectors/dnscheck/dnscheck.go index 8c87fbf673cd..11769142787d 100644 --- a/pkg/detectors/dnscheck/dnscheck.go +++ b/pkg/detectors/dnscheck/dnscheck.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/docker/docker_auth_config.go b/pkg/detectors/docker/docker_auth_config.go index 6a6fae1a3125..eced529e61ab 100644 --- a/pkg/detectors/docker/docker_auth_config.go +++ b/pkg/detectors/docker/docker_auth_config.go @@ -48,7 +48,7 @@ func (s Scanner) MaxSecretSize() int64 { } var ( - keyPat = regexp.MustCompile(`{(?:\s|\\+[nrt])*\\*"auths\\*"(?:\s|\\+t)*:(?:\s|\\+t)*{(?:\s|\\+[nrt])*\\*"(?i:https?:\/\/)?[a-z0-9\-.:\/]+\\*"(?:\s|\\+t)*:(?:\s|\\+t)*{(?:(?:\s|\\+[nrt])*\\*"(?i:auth|email|username|password)\\*"\s*:\s*\\*".*\\*"\s*,?)+?(?:\s|\\+[nrt])*}(?:\s|\\+[nrt])*}(?:\s|\\+[nrt])*}`) + keyPat = regexp.MustCompile(`{(?:\s|\\+[nrt])*\\*"auths\\*"(?:\s|\\+t)*:(?:\s|\\+t)*{(?:\s|\\+[nrt])*\\*"(?i:https?:\/\/)?[a-z0-9\-.:\/]+\\*"(?:\s|\\+t)*:(?:\s|\\+t)*{(?:(?:\s|\\+[nrt])*\\*"(?i:auth|email|username|password)\\*"\s*:\s*\\*".*?\\*"\s*,?)+?(?:\s|\\+[nrt])*}(?:\s|\\+[nrt])*}(?:\s|\\+[nrt])*}`) escapedReplacer = strings.NewReplacer( `\n`, "", `\r`, "", @@ -188,7 +188,12 @@ func verifyMatch(ctx logContext.Context, client *http.Client, registry string, u return false, nil } - if !strings.HasPrefix(h, "Bearer") { + scheme, _, _ := strings.Cut(h, " ") + if strings.EqualFold(scheme, "Basic") { + return false, nil + } + + if !strings.EqualFold(scheme, "Bearer") { return false, fmt.Errorf("unsupported WWW-Authenticate auth scheme: %s", h) } diff --git a/pkg/detectors/docker/docker_auth_config_test.go b/pkg/detectors/docker/docker_auth_config_test.go index 67ff7e7ff1b7..6366fb952940 100644 --- a/pkg/detectors/docker/docker_auth_config_test.go +++ b/pkg/detectors/docker/docker_auth_config_test.go @@ -1,11 +1,14 @@ package docker import ( + "net/http" + "net/http/httptest" + "testing" + "github.com/google/go-cmp/cmp" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" - "testing" ) func TestDocker_Pattern(t *testing.T) { @@ -103,10 +106,18 @@ func TestDocker_Pattern(t *testing.T) { { name: "content after last }", // This is base64-encoded, however, that doesn't get detected in these tests. - //input: `{"kind":"AdmissionReview","apiVersion":"admission.k8s.io/v1","request":{"uid":"b9d17c49-1b2c-421a-8ae8-3b3d252d2f61","kind":{"group":"","version":"v1","kind":"Secret"},"resource":{"group":"","version":"v1","resource":"secrets"},"requestKind":{"group":"","version":"v1","kind":"Secret"},"requestResource":{"group":"","version":"v1","resource":"secrets"},"name":"regcred","namespace":"test-webhooks","operation":"CREATE","userInfo":{"username":"kube:admin","groups":["system:cluster-admins","system:authenticated"],"extra":{"scopes.authorization.openshift.io":["user:full"]}},"object":{"kind":"Secret","apiVersion":"v1","metadata":{"name":"regcred","namespace":"test-webhooks","uid":"544674ac-f0fb-4a30-994b-eab579e1f418","creationTimestamp":"2022-05-03T15:16:55Z","managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2022-05-03T15:16:55Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:.dockerconfigjson":{}},"f:type":{}}}]},"data":{".dockerconfigjson":"eyJhdXRocyI6eyJxdWF5LmlvIjp7InVzZXJuYW1lIjoiMTIzIiwicGFzc3dvcmQiOiIxMjMiLCJhdXRoIjoiTVRJek9qRXlNdz09In19fQ=="},"type":"kubernetes.io/dockerconfigjson"},"oldObject":null,"dryRun":false,"options":{"kind":"CreateOptions","apiVersion":"meta.k8s.io/v1","fieldManager":"kubectl-create"}}}`, + // input: `{"kind":"AdmissionReview","apiVersion":"admission.k8s.io/v1","request":{"uid":"b9d17c49-1b2c-421a-8ae8-3b3d252d2f61","kind":{"group":"","version":"v1","kind":"Secret"},"resource":{"group":"","version":"v1","resource":"secrets"},"requestKind":{"group":"","version":"v1","kind":"Secret"},"requestResource":{"group":"","version":"v1","resource":"secrets"},"name":"regcred","namespace":"test-webhooks","operation":"CREATE","userInfo":{"username":"kube:admin","groups":["system:cluster-admins","system:authenticated"],"extra":{"scopes.authorization.openshift.io":["user:full"]}},"object":{"kind":"Secret","apiVersion":"v1","metadata":{"name":"regcred","namespace":"test-webhooks","uid":"544674ac-f0fb-4a30-994b-eab579e1f418","creationTimestamp":"2022-05-03T15:16:55Z","managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2022-05-03T15:16:55Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:.dockerconfigjson":{}},"f:type":{}}}]},"data":{".dockerconfigjson":"eyJhdXRocyI6eyJxdWF5LmlvIjp7InVzZXJuYW1lIjoiMTIzIiwicGFzc3dvcmQiOiIxMjMiLCJhdXRoIjoiTVRJek9qRXlNdz09In19fQ=="},"type":"kubernetes.io/dockerconfigjson"},"oldObject":null,"dryRun":false,"options":{"kind":"CreateOptions","apiVersion":"meta.k8s.io/v1","fieldManager":"kubectl-create"}}}`, input: `{"kind":"AdmissionReview","apiVersion":"admission.k8s.io/v1","request":{"uid":"b9d17c49-1b2c-421a-8ae8-3b3d252d2f61","kind":{"group":"","version":"v1","kind":"Secret"},"resource":{"group":"","version":"v1","resource":"secrets"},"requestKind":{"group":"","version":"v1","kind":"Secret"},"requestResource":{"group":"","version":"v1","resource":"secrets"},"name":"regcred","namespace":"test-webhooks","operation":"CREATE","userInfo":{"username":"kube:admin","groups":["system:cluster-admins","system:authenticated"],"extra":{"scopes.authorization.openshift.io":["user:full"]}},"object":{"kind":"Secret","apiVersion":"v1","metadata":{"name":"regcred","namespace":"test-webhooks","uid":"544674ac-f0fb-4a30-994b-eab579e1f418","creationTimestamp":"2022-05-03T15:16:55Z","managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2022-05-03T15:16:55Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:.dockerconfigjson":{}},"f:type":{}}}]},"data":{".dockerconfigjson":"{"auths":{"quay.io":{"username":"123","password":"123","auth":"MTIzOjEyMw=="}}}"},"type":"kubernetes.io/dockerconfigjson"},"oldObject":null,"dryRun":false,"options":{"kind":"CreateOptions","apiVersion":"meta.k8s.io/v1","fieldManager":"kubectl-create"}}}`, want: []string{`{"registry":"quay.io","auth":"MTIzOjEyMw=="}`}, }, + { + name: "extra matches on one line", + input: `{"auths":{"https://index.docker.io/v2/":{"auth":"Y29uZXhwOkZUQUBDTkNGMG5AenVyZTM="}}} anything at all; {"auths":{"https://index.docker.io/v2/":{"auth":"dHJ1ZmZsZWhvZzpiZDQyNzQ2Yy1hNzc3LTQ4ZDktYjBhMi04N2I2YzEzMjdkMDA="}}}`, + want: []string{ + `{"registry":"https://index.docker.io/v2/","auth":"Y29uZXhwOkZUQUBDTkNGMG5AenVyZTM="}`, + `{"registry":"https://index.docker.io/v2/","auth":"dHJ1ZmZsZWhvZzpiZDQyNzQ2Yy1hNzc3LTQ4ZDktYjBhMi04N2I2YzEzMjdkMDA="}`, + }, + }, // False-positives { @@ -291,3 +302,102 @@ func Test_ParseAuthenticateHeader(t *testing.T) { } } } + +func Test_VerifyMatch(t *testing.T) { + // base64 of "user:pass" + const basicAuth = "dXNlcjpwYXNz" + + // bearerChallenge points the token realm back at the test server so the follow-up + // request stays in-process. + bearerChallenge := func(host string) string { + return `Bearer realm="http://` + host + `/token",service="registry"` + } + + tests := []struct { + name string + handler http.HandlerFunc + wantVerified bool + wantErr bool + }{ + { + name: "harbor rejects basic auth", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Www-Authenticate", `Basic realm="harbor"`) + w.WriteHeader(http.StatusUnauthorized) + }, + wantVerified: false, + wantErr: false, + }, + { + name: "unauthorized without a challenge", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }, + wantVerified: false, + wantErr: false, + }, + { + name: "unsupported scheme is still indeterminate", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Www-Authenticate", `Digest realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + }, + wantVerified: false, + wantErr: true, + }, + { + name: "bearer token exchange rejects credentials", + handler: func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Www-Authenticate", bearerChallenge(r.Host)) + w.WriteHeader(http.StatusUnauthorized) + }, + wantVerified: false, + wantErr: false, + }, + { + name: "bearer token exchange accepts credentials", + handler: func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Www-Authenticate", bearerChallenge(r.Host)) + w.WriteHeader(http.StatusUnauthorized) + }, + wantVerified: true, + wantErr: false, + }, + { + name: "registry accepts basic auth", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + }, + wantVerified: true, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(tc.handler) + defer server.Close() + + verified, err := verifyMatch(context.Background(), server.Client(), server.URL, "user", basicAuth) + + if tc.wantErr && err == nil { + t.Errorf("expected a verification error, got nil") + } + if !tc.wantErr && err != nil { + t.Errorf("expected no verification error, got %v", err) + } + if verified != tc.wantVerified { + t.Errorf("verified = %v, want %v", verified, tc.wantVerified) + } + }) + } +} diff --git a/pkg/detectors/dockerhub/v1/dockerhub.go b/pkg/detectors/dockerhub/v1/dockerhub.go index e29ce4c239c0..f28d77f3230e 100644 --- a/pkg/detectors/dockerhub/v1/dockerhub.go +++ b/pkg/detectors/dockerhub/v1/dockerhub.go @@ -64,13 +64,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // Process results. for token := range tokens { - s1 := detectors.Result{ - DetectorType: s.Type(), - Raw: []byte(token), - } - for username := range usernames { - s1.RawV2 = []byte(fmt.Sprintf("%s:%s", username, token)) + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(token), + RawV2: []byte(fmt.Sprintf("%s:%s", username, token)), + SecretParts: map[string]string{ + "pat": token, + "username": username, + }, + } if verify { if s.client == nil { @@ -81,12 +84,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr) - if s1.Verified { - s1.SecretParts = map[string]string{ - "username": username, - "pat": token, - } - } } results = append(results, s1) @@ -98,7 +95,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // PAT matches without usernames cannot be verified but might still be useful. if len(usernames) == 0 { - results = append(results, s1) + results = append(results, detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(token), + SecretParts: map[string]string{"pat": token}, + }) } } return @@ -117,13 +118,14 @@ func (s Scanner) verifyMatch(ctx context.Context, username string, password stri if err != nil { return false, nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() body, err := io.ReadAll(res.Body) if err != nil { return false, nil, err } - if res.StatusCode == http.StatusOK { + switch res.StatusCode { + case http.StatusOK: var tokenRes tokenResponse if err := json.Unmarshal(body, &tokenRes); (err != nil || tokenRes == tokenResponse{}) { return false, nil, err @@ -144,7 +146,7 @@ func (s Scanner) verifyMatch(ctx context.Context, username string, password stri return true, extraData, nil } return true, nil, nil - } else if res.StatusCode == http.StatusUnauthorized { + case http.StatusUnauthorized: // Valid credentials can still return a 401 status code if 2FA is enabled var mfaRes mfaRequiredResponse if err := json.Unmarshal(body, &mfaRes); err != nil || mfaRes.MfaToken == "" { @@ -156,7 +158,7 @@ func (s Scanner) verifyMatch(ctx context.Context, username string, password stri "2fa_required": "true", } return true, extraData, nil - } else { + default: return false, nil, fmt.Errorf("unexpected response status %d", res.StatusCode) } } diff --git a/pkg/detectors/dockerhub/v2/dockerhub.go b/pkg/detectors/dockerhub/v2/dockerhub.go index 4f55f6fa91c6..544a86d6040c 100644 --- a/pkg/detectors/dockerhub/v2/dockerhub.go +++ b/pkg/detectors/dockerhub/v2/dockerhub.go @@ -64,13 +64,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // Process results. for token := range tokens { - s1 := detectors.Result{ - DetectorType: s.Type(), - Raw: []byte(token), - } - for username := range usernames { - s1.RawV2 = []byte(fmt.Sprintf("%s:%s", username, token)) + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(token), + RawV2: []byte(fmt.Sprintf("%s:%s", username, token)), + SecretParts: map[string]string{ + "pat": token, + "username": username, + }, + } if verify { if s.client == nil { @@ -81,12 +84,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr) - if s1.Verified { - s1.SecretParts = map[string]string{ - "username": username, - "pat": token, - } - } } results = append(results, s1) @@ -98,7 +95,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // PAT matches without usernames cannot be verified but might still be useful. if len(usernames) == 0 { - results = append(results, s1) + results = append(results, detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(token), + SecretParts: map[string]string{"pat": token}, + }) } } return @@ -117,13 +118,14 @@ func (s Scanner) verifyMatch(ctx context.Context, username string, password stri if err != nil { return false, nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() body, err := io.ReadAll(res.Body) if err != nil { return false, nil, err } - if res.StatusCode == http.StatusOK { + switch res.StatusCode { + case http.StatusOK: var tokenRes tokenResponse if err := json.Unmarshal(body, &tokenRes); (err != nil || tokenRes == tokenResponse{}) { return false, nil, err @@ -144,7 +146,7 @@ func (s Scanner) verifyMatch(ctx context.Context, username string, password stri return true, extraData, nil } return true, nil, nil - } else if res.StatusCode == http.StatusUnauthorized { + case http.StatusUnauthorized: // Valid credentials can still return a 401 status code if 2FA is enabled var mfaRes mfaRequiredResponse if err := json.Unmarshal(body, &mfaRes); err != nil || mfaRes.MfaToken == "" { @@ -156,7 +158,7 @@ func (s Scanner) verifyMatch(ctx context.Context, username string, password stri "2fa_required": "true", } return true, extraData, nil - } else { + default: return false, nil, fmt.Errorf("unexpected response status %d", res.StatusCode) } } diff --git a/pkg/detectors/docparser/docparser.go b/pkg/detectors/docparser/docparser.go index f8b1bc825027..01a98131bea4 100644 --- a/pkg/detectors/docparser/docparser.go +++ b/pkg/detectors/docparser/docparser.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/documo/documo.go b/pkg/detectors/documo/documo.go index 461836511bae..8a2dab35143c 100644 --- a/pkg/detectors/documo/documo.go +++ b/pkg/detectors/documo/documo.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/docusign/docusign.go b/pkg/detectors/docusign/docusign.go index aacd1e061505..dc57157384d0 100644 --- a/pkg/detectors/docusign/docusign.go +++ b/pkg/detectors/docusign/docusign.go @@ -83,7 +83,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "ey") - res.Body.Close() + _ = res.Body.Close() if err != nil { return nil, err diff --git a/pkg/detectors/doppler/doppler.go b/pkg/detectors/doppler/doppler.go index d711d846aea0..32e240e70dc6 100644 --- a/pkg/detectors/doppler/doppler.go +++ b/pkg/detectors/doppler/doppler.go @@ -73,7 +73,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true var r response diff --git a/pkg/detectors/dronahq/dronahq.go b/pkg/detectors/dronahq/dronahq.go index 83d24f63de92..63166205e6b7 100644 --- a/pkg/detectors/dronahq/dronahq.go +++ b/pkg/detectors/dronahq/dronahq.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/droneci/droneci.go b/pkg/detectors/droneci/droneci.go index e16609ad4a3a..7bc3e6df704d 100644 --- a/pkg/detectors/droneci/droneci.go +++ b/pkg/detectors/droneci/droneci.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/dropbox/dropbox.go b/pkg/detectors/dropbox/dropbox.go index 82438c8dac1b..503777b1862d 100644 --- a/pkg/detectors/dropbox/dropbox.go +++ b/pkg/detectors/dropbox/dropbox.go @@ -20,6 +20,12 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) + +// MaxSecretSize overrides the engine's default keyword window (512 bytes) so the full +// token is passed to FromData. Newer scoped Dropbox short-lived tokens (sl.u.…) can be +// ~1.5KB; without this the engine truncates the chunk window and verification fails. +func (s Scanner) MaxSecretSize() int64 { return 4096 } var ( defaultClient = common.SaneHttpClient() @@ -46,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Dropbox, Raw: []byte(key), + SecretParts: map[string]string{"token": key}, } if verify { @@ -57,9 +64,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyDropboxToken(ctx, client, key) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - if s1.Verified { - s1.SecretParts = map[string]string{"token": key} - } } results = append(results, s1) diff --git a/pkg/detectors/dropbox/dropbox_test.go b/pkg/detectors/dropbox/dropbox_test.go index 990284e3ce22..6a584d1e6d88 100644 --- a/pkg/detectors/dropbox/dropbox_test.go +++ b/pkg/detectors/dropbox/dropbox_test.go @@ -2,6 +2,7 @@ package dropbox import ( "context" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -92,3 +93,38 @@ func TestDropBox_Pattern(t *testing.T) { }) } } + +// TestDropBox_LongTokenThroughEngineWindow guards the MaxSecretSize override. The scanning +// engine only passes a keyword-centered window of the chunk to FromData (512 bytes by +// default). Newer scoped Dropbox tokens (sl.u.…) can be ~1.5KB, so without MaxSecretSize the +// token is truncated before the regex sees it and verification fails. This exercises the full +// Aho-Corasick windowing path rather than calling FromData on the whole input directly. +func TestDropBox_LongTokenThroughEngineWindow(t *testing.T) { + token := "sl.u." + strings.Repeat("aB3-_", 300) // 5 + 1500 = 1505 chars, single base64url run + chunk := []byte("DROPBOX_TOKEN=" + token + "\n") + + d := Scanner{} + core := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + matches := core.FindDetectorMatches(chunk) + if len(matches) == 0 { + t.Fatal("no detector matches for long token") + } + + var found bool + for _, m := range matches { + for _, data := range m.Matches() { + results, err := d.FromData(context.Background(), false, data) + if err != nil { + t.Fatal(err) + } + for _, r := range results { + if string(r.Raw) == token { + found = true + } + } + } + } + if !found { + t.Errorf("full %d-char token was not captured through the engine window (MaxSecretSize regression?)", len(token)) + } +} diff --git a/pkg/detectors/duffeltoken/duffeltoken.go b/pkg/detectors/duffeltoken/duffeltoken.go new file mode 100644 index 000000000000..aafca76dcc37 --- /dev/null +++ b/pkg/detectors/duffeltoken/duffeltoken.go @@ -0,0 +1,144 @@ +package duffeltoken + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient()) + + // Duffel test token pattern + // Format: (duffel_test_ or duffel_live_) + 43 alphanumeric / dash / underscore characters + duffelTestTokenPat = regexp.MustCompile( + `\b(duffel_(test|live)_[A-Za-z0-9_-]{43})(?:$|[^A-Za-z0-9_-])`, + ) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{"duffel_test_", "duffel_live_"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for Duffel test tokens and optionally verifies them +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range duffelTestTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_DuffelToken, + Raw: []byte(token), + Redacted: token[:15] + "...", + SecretParts: map[string]string{ + "token": token, + }, + } + + if verify { + verified, verificationErr := verifyDuffelToken( + ctx, + s.getClient(), + token, + ) + + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + + return +} + +func verifyDuffelToken( + ctx context.Context, + client *http.Client, + token string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://api.duffel.com/identity/customer/users?limit=1", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Duffel-Version", "v2") + req.Header.Set("Accept", "application/json") + + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_DuffelToken, token, req) + + if err != nil { + return false, err + } + + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + + case http.StatusOK: + return true, nil + + case http.StatusUnauthorized: + // Token invalid or revoked + return false, nil + + case http.StatusForbidden: + // Token valid but insufficient permissions - treat as verified since the token is active + return true, nil + + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_DuffelToken +} + +func (s Scanner) Description() string { + return "Duffel is a flight search and booking API service. Duffel API tokens can be used to access and interact with flight search and booking APIs in test environments." +} diff --git a/pkg/detectors/duffeltoken/duffeltoken_integration_test.go b/pkg/detectors/duffeltoken/duffeltoken_integration_test.go new file mode 100644 index 000000000000..fc7c411dfe1a --- /dev/null +++ b/pkg/detectors/duffeltoken/duffeltoken_integration_test.go @@ -0,0 +1,180 @@ +//go:build detectors +// +build detectors + +package duffeltoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestDuffelTestToken_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeToken := testSecrets.MustGetField("DUFFEL_TEST_TOKEN") + + inactiveToken := "duffel_test_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Duffel token %s for API", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_DuffelToken, + Verified: true, + Raw: []byte(activeToken), + }, + }, + }, + { + name: "found, real token, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Duffel token %s for API", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_DuffelToken, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Duffel token %s for API", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_DuffelToken, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Duffel token %s for API", inactiveToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_DuffelToken, + Verified: false, + Raw: []byte(inactiveToken), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("DuffelTestToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "Redacted", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("DuffelTestToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkDuffelTestToken_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/duffeltoken/duffeltoken_test.go b/pkg/detectors/duffeltoken/duffeltoken_test.go new file mode 100644 index 000000000000..4f2c37ac7627 --- /dev/null +++ b/pkg/detectors/duffeltoken/duffeltoken_test.go @@ -0,0 +1,150 @@ +package duffeltoken + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestDuffelTestToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: ` + func setupDuffelClient() (*http.Client, error) { + url := "https://api.duffel.com/identity/customer/users?limit=1" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return nil, err + } + + duffelToken := "duffel_test_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + req.Header.Set("Authorization", "Bearer "+duffelToken) + req.Header.Set("Duffel-Version", "v2") + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer func() { _ = resp.Body.Close() }() + + return client, nil + } + `, + want: []string{ + "duffel_test_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + { + name: "valid pattern - env file", + input: ` + # .env.production + APP_NAME=flight-booker + APP_ENV=production + DUFFEL_API_TOKEN=duffel_test_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + LOG_LEVEL=info + `, + want: []string{ + "duffel_test_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + func loadDuffelTokens() map[string]string { + // Tokens for each deployment environment + return map[string]string{ + "staging": "duffel_test_ccccccccccccccccccccccccccccccccccccccccccc", + "production": "duffel_live_ddddddddddddddddddddddddddddddddddddddddddd", + } + } + `, + want: []string{ + "duffel_test_ccccccccccccccccccccccccccccccccccccccccccc", + "duffel_live_ddddddddddddddddddddddddddddddddddddddddddd", + }, + }, + { + name: "invalid pattern - too short", + input: ` + func setupDuffelClient() { + // Truncated token accidentally committed + token := "duffel_test_abc123" + client.SetAuthToken(token) + } + `, + want: nil, + }, + { + name: "invalid pattern - invalid characters", + input: ` + func setupDuffelClient() { + // Token contains an invalid special character + token := "duffel_test_aaaaaaaaaaaaaaaaaaaaaaa!aaaaaaaaaaaaaaaaaaa" + client.SetAuthToken(token) + } + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + // TODO: replace hardcoded duffel_test_ prefix check with proper validation + func isDuffelTestToken(s string) bool { + return strings.HasPrefix(s, "duffel_test_") + } + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords()) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results)) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/duo/duo.go b/pkg/detectors/duo/duo.go new file mode 100644 index 000000000000..664bb97f7eb6 --- /dev/null +++ b/pkg/detectors/duo/duo.go @@ -0,0 +1,221 @@ +package duo + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" + "time" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // Integration key is of 20 characters with only capital alphabets and digits. + integrationKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"duo"}) + `\b(DI[A-Z0-9]{18})\b`) + + // Secret key is of 40 characters with only alphabets and digits. + secretKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"duo"}) + `\b([A-Za-z0-9]{40})\b`) + + // Host is usually a subdomain of duosecurity.com e.g api-21321awda.duosecurity.com + apiHost = regexp.MustCompile(`\b([a-z0-9-]{6,}\.duosecurity\.com)\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"duosecurity"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData will find and optionally verify DeepSeek secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueHosts := make(map[string]struct{}) + uniqueIntKeys := make(map[string]struct{}) + uniqueSecretKeys := make(map[string]struct{}) + for _, match := range apiHost.FindAllStringSubmatch(dataStr, -1) { + uniqueHosts[match[1]] = struct{}{} + } + for _, match := range integrationKeyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueIntKeys[match[1]] = struct{}{} + } + for _, match := range secretKeyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueSecretKeys[match[1]] = struct{}{} + } + for host := range uniqueHosts { + for apiKey := range uniqueIntKeys { + for apiSecret := range uniqueSecretKeys { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_Duo, + Raw: []byte(apiKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, apiKey, apiSecret)), + ExtraData: map[string]string{ + "application": "Admin API", + }, + SecretParts: map[string]string{ + "host": host, + "ikey": apiKey, + "skey": apiSecret, + "application": "Admin API", + }, + } + if verify { + verified, verificationErr := VerifyAdminToken(ctx, s.getClient(), host, apiKey, apiSecret) + // we will try auth api verification if admin api verification fails without error, + // which is in the case of it returning 401 unauthorized due to invalid credentials. + if !verified && verificationErr == nil { + var authStatusCode int + verified, authStatusCode, verificationErr = VerifyAuthToken(ctx, s.getClient(), host, apiKey, apiSecret) + // The Auth API responds 403 (not 401) when ikey/skey are + // structurally valid Admin API keys, active or not. That still + // counts as verified, but the keys remain Admin API keys, so + // only relabel to Auth API when the check actually succeeded (200). + if authStatusCode != http.StatusForbidden { + s1.ExtraData["application"] = "Auth API" + s1.SecretParts["application"] = "Auth API" + } + } + s1.SetVerificationError(verificationErr, host, apiKey, apiSecret) + s1.Verified = verified + } + results = append(results, s1) + } + } + } + return results, nil +} + +// returns verified=true if credentials are valid and belong to auth api, false if creds are invalid, and error if creds belong to auth api or for anything else (e.g., network error). +// The HTTP status code is also returned so callers can distinguish a genuine 200 from a +// 403, which the Auth API returns for structurally valid Admin API keys (active or not). +func VerifyAuthToken( + ctx context.Context, + client *http.Client, + host, ikey, skey string, +) (bool, int, error) { + // Docs: https://duo.com/docs/authapi#check + return verifyDuoRequest( + ctx, + client, + host, + ikey, + skey, + "/auth/v2/check", + ) +} + +// returns 401 unauthorized if credentials are invalid, 200 OK if valid, and error for anything else +func VerifyAdminToken( + ctx context.Context, + client *http.Client, + host, ikey, skey string, +) (bool, error) { + // Docs: https://duo.com/docs/adminapi#account-info + verified, _, err := verifyDuoRequest( + ctx, + client, + host, + ikey, + skey, + "/admin/v1/info/summary", + ) + return verified, err +} + +func verifyDuoRequest( + ctx context.Context, + client *http.Client, + host, ikey, skey string, + path string, +) (bool, int, error) { + + timestamp := time.Now().UTC().Format(http.TimeFormat) + + // Canonical request string + canonical := strings.Join([]string{ + timestamp, + http.MethodGet, + host, + path, + "", + }, "\n") + + // HMAC signature + mac := hmac.New(sha1.New, []byte(skey)) + _, _ = mac.Write([]byte(canonical)) + signature := hex.EncodeToString(mac.Sum(nil)) + + // Authorization header + auth := base64.StdEncoding.EncodeToString( + []byte(ikey + ":" + signature), + ) + + // Build request + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://"+host+path, + nil, + ) + if err != nil { + return false, 0, err + } + + req.Header.Set("Authorization", "Basic "+auth) + req.Header.Set("Date", timestamp) + + res, err := client.Do(req) + if err != nil { + return false, 0, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, res.StatusCode, nil + case http.StatusUnauthorized: + return false, res.StatusCode, nil + case http.StatusForbidden: // Auth API returns 403 if credentials are admin keys be it active or inactive + return true, res.StatusCode, nil + default: + return false, res.StatusCode, fmt.Errorf("unexpected HTTP status %d", res.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_Duo +} + +func (s Scanner) Description() string { + return "Duo is a security platform that provides multi-factor authentication and identity management services." +} diff --git a/pkg/detectors/duo/duo_integration_test.go b/pkg/detectors/duo/duo_integration_test.go new file mode 100644 index 000000000000..8d3ef7fc9fe5 --- /dev/null +++ b/pkg/detectors/duo/duo_integration_test.go @@ -0,0 +1,310 @@ +//go:build detectors +// +build detectors + +package duo + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestDuo_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + host := testSecrets.MustGetField("DUO_APIHOST") + ikey := testSecrets.MustGetField("DUO_AUTH_IKEY") + skey := testSecrets.MustGetField("DUO_AUTH_SKEY") + inactiveSKey := "CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using DUO_APIHOST=%s DUO_IKEY=%s DUO_SKEY=%s", + host, ikey, skey, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Duo, + Verified: true, + Raw: []byte(ikey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, ikey, skey)), + }, + }, + }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "DUO_APIHOST=%s DUO_IKEY=%s DUO_SKEY=%s", + host, ikey, skey, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Duo, + Verified: false, + Raw: []byte(ikey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, ikey, skey)), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "DUO_APIHOST=%s DUO_IKEY=%s DUO_SKEY=%s", + host, ikey, skey, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Duo, + Verified: false, + Raw: []byte(ikey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, ikey, skey)), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive secret)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "DUO_APIHOST=%s DUO_IKEY=%s DUO_SKEY=%s", + host, ikey, inactiveSKey, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Duo, + Verified: false, + Raw: []byte(ikey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, ikey, inactiveSKey)), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no duo credentials here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("Duo.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError=%v, verificationError=%v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Duo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func TestDuo_FromChunk_AdminAPI(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + // Auth API credentials + host := testSecrets.MustGetField("DUO_APIHOST") + inactiveSKey := "CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe" + inactiveIKey := "DIBJ9FGFZDD8BQKF6PN1" + + // Admin API credentials + adminIKey := testSecrets.MustGetField("DUO_ADMIN_IKEY") + adminSKey := testSecrets.MustGetField("DUO_ADMIN_SKEY") + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "admin key, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using DUO_APIHOST=%s DUO_IKEY=%s DUO_SKEY=%s", + host, adminIKey, adminSKey, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Duo, + Verified: true, + Raw: []byte(adminIKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, adminIKey, adminSKey)), + ExtraData: map[string]string{ + "application": "Admin API", + }, + }, + }, + }, + { + name: "admin key, unverified (invalid secret)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using DUO_APIHOST=%s DUO_IKEY=%s DUO_SKEY=%s", + host, inactiveIKey, inactiveSKey, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Duo, + Verified: false, + Raw: []byte(inactiveIKey), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", host, inactiveIKey, inactiveSKey)), + ExtraData: map[string]string{ + "application": "Auth API", + }, + }, + }, + wantVerificationErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("Duo.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError=%v, verificationError=%v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "verificationError", + "primarySecret", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Duo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/duo/duo_test.go b/pkg/detectors/duo/duo_test.go new file mode 100644 index 000000000000..1160d911e7d5 --- /dev/null +++ b/pkg/detectors/duo/duo_test.go @@ -0,0 +1,163 @@ +package duo + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestDuo_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - auth api credentials", + input: ` + [INFO] Initializing Duo Auth API + [DEBUG] DUO_API_HOST=api-123456.duosecurity.com + [DEBUG] DUO_IKEY=DIXXXXXXXXXXXXXXXXXX + [DEBUG] DUO_SKEY=CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe + `, + want: []string{ + "api-123456.duosecurity.com:DIXXXXXXXXXXXXXXXXXX:CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe", + }, + }, + { + name: "valid pattern - admin api credentials", + input: ` + [INFO] Connecting to Duo Admin API + [DEBUG] duo_host=api-abcdef.duosecurity.com + [DEBUG] duo_integration_key=DIYYYYYYYYYYYYYYYYYY + [DEBUG] duo_secret_key=CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe + `, + want: []string{ + "api-abcdef.duosecurity.com:DIYYYYYYYYYYYYYYYYYY:CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe", + }, + }, + { + name: "valid pattern - json config", + input: ` + { + "duo": { + "host": "api-zzzzzz.duosecurity.com", + "ikey": "DIABCDEFGHIJKLMNOPQR", + "duo_skey": "CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe" + } + } + `, + want: []string{ + "api-zzzzzz.duosecurity.com:DIABCDEFGHIJKLMNOPQR:CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe", + }, + }, + { + name: "invalid pattern - only ikey", + input: ` + [DEBUG] DUO_IKEY=DIXXXXXXXXXXXXXXXXXX + `, + want: nil, + }, + { + name: "invalid pattern - only skey", + input: ` + [DEBUG] DUO_SKEY=CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe + `, + want: nil, + }, + { + name: "invalid pattern - only host", + input: ` + [DEBUG] DUO_API_HOST=api-123456.duosecurity.com + `, + want: nil, + }, + { + name: "invalid pattern - missing host", + input: ` + [DEBUG] DUO_IKEY=DIXXXXXXXXXXXXXXXXXX + [DEBUG] DUO_SKEY=CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe + `, + want: nil, + }, + { + name: "invalid pattern - short ikey", + input: ` + [DEBUG] DUO_API_HOST=api-123456.duosecurity.com + [DEBUG] DUO_IKEY=DI123 + [DEBUG] DUO_SKEY=CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe + `, + want: nil, + }, + { + name: "invalid pattern - short skey", + input: ` + [DEBUG] DUO_API_HOST=api-123456.duosecurity.com + [DEBUG] DUO_IKEY=DIXXXXXXXXXXXXXXXXXX + [DEBUG] DUO_SKEY=deadbeef + `, + want: nil, + }, + { + name: "invalid pattern - wrong host domain", + input: ` + [DEBUG] DUO_API_HOST=duo.example.com + [DEBUG] DUO_IKEY=DIXXXXXXXXXXXXXXXXXX + [DEBUG] DUO_SKEY=CWZZCIOF2aEHdx2PfexiNC3Bedai2axLMC3C2IFe + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + if len(test.want) > 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found", + test.name, + d.Keywords(), + ) + } + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + if len(results) != len(test.want) { + t.Fatalf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/duply/duply.go b/pkg/detectors/duply/duply.go index 03a1574cda2f..c83d4a34b526 100644 --- a/pkg/detectors/duply/duply.go +++ b/pkg/detectors/duply/duply.go @@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/dynalist/dynalist.go b/pkg/detectors/dynalist/dynalist.go index 21d69765cf09..c734571f2373 100644 --- a/pkg/detectors/dynalist/dynalist.go +++ b/pkg/detectors/dynalist/dynalist.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"_code":"Ok"`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/dyspatch/dyspatch.go b/pkg/detectors/dyspatch/dyspatch.go index f4772b3d4a65..3842b4588de6 100644 --- a/pkg/detectors/dyspatch/dyspatch.go +++ b/pkg/detectors/dyspatch/dyspatch.go @@ -48,26 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.dyspatch.io/templates", nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/vnd.dyspatch.2020.11+json") - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - validResponse := strings.Contains(body, "limited_usage") || strings.Contains(body, "data") - - if validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -76,6 +59,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.dyspatch.io/templates", nil) + if err != nil { + return false, err + } + req.Header.Add("Accept", "application/vnd.dyspatch.2020.11+json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + validResponse := strings.Contains(body, "limited_usage") || strings.Contains(body, "data") + + if validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Dyspatch } diff --git a/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go b/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go index af48b0415893..6e4d600d631b 100644 --- a/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go +++ b/pkg/detectors/eagleeyenetworks/eagleeyenetworks.go @@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ecostruxureit/ecostruxureit.go b/pkg/detectors/ecostruxureit/ecostruxureit.go index 24e143ce0a91..f70caa4e8670 100644 --- a/pkg/detectors/ecostruxureit/ecostruxureit.go +++ b/pkg/detectors/ecostruxureit/ecostruxureit.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/edamam/edamam.go b/pkg/detectors/edamam/edamam.go index afb6edceb8b2..77563cc2ad4f 100644 --- a/pkg/detectors/edamam/edamam.go +++ b/pkg/detectors/edamam/edamam.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/edenai/edenai.go b/pkg/detectors/edenai/edenai.go index d43d83ac021d..16c563dc061f 100644 --- a/pkg/detectors/edenai/edenai.go +++ b/pkg/detectors/edenai/edenai.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/eightxeight/eightxeight.go b/pkg/detectors/eightxeight/eightxeight.go index 6543a86e299b..7df28a91ab23 100644 --- a/pkg/detectors/eightxeight/eightxeight.go +++ b/pkg/detectors/eightxeight/eightxeight.go @@ -71,7 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/elasticemail/elasticemail.go b/pkg/detectors/elasticemail/elasticemail.go index d49470b4f70e..0225a843166e 100644 --- a/pkg/detectors/elasticemail/elasticemail.go +++ b/pkg/detectors/elasticemail/elasticemail.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { data, readErr := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() if readErr == nil { var ResVar struct { Success bool `json:"success"` diff --git a/pkg/detectors/elevenlabs/v1/elevenlabs.go b/pkg/detectors/elevenlabs/v1/elevenlabs.go index 7a559f769935..669c1b1697b2 100644 --- a/pkg/detectors/elevenlabs/v1/elevenlabs.go +++ b/pkg/detectors/elevenlabs/v1/elevenlabs.go @@ -59,6 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "version": "1", "rotation_guide": "https://howtorotate.com/docs/tutorials/elevenlabs/", }, + SecretParts: map[string]string{"key": match}, } if verify { @@ -74,12 +75,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["Tier"] = userResponse.Subscription.Tier } s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": match, - } - } } results = append(results, s1) diff --git a/pkg/detectors/elevenlabs/v2/elevenlabs.go b/pkg/detectors/elevenlabs/v2/elevenlabs.go index b7279f2bd6f6..146a71055f56 100644 --- a/pkg/detectors/elevenlabs/v2/elevenlabs.go +++ b/pkg/detectors/elevenlabs/v2/elevenlabs.go @@ -56,6 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_ElevenLabs, Raw: []byte(match), ExtraData: map[string]string{"version": "2"}, + SecretParts: map[string]string{"key": match}, } if verify { @@ -71,12 +72,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["Tier"] = userResponse.Subscription.Tier } s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": match, - } - } } results = append(results, s1) diff --git a/pkg/detectors/enablex/enablex.go b/pkg/detectors/enablex/enablex.go index 2fbf690c9c6c..ca1e0c9c686e 100644 --- a/pkg/detectors/enablex/enablex.go +++ b/pkg/detectors/enablex/enablex.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/enigma/enigma.go b/pkg/detectors/enigma/enigma.go index 18df615e0865..f8a9e76b5d51 100644 --- a/pkg/detectors/enigma/enigma.go +++ b/pkg/detectors/enigma/enigma.go @@ -2,6 +2,8 @@ package enigma import ( "context" + "fmt" + "io" "net/http" "strings" @@ -33,7 +35,6 @@ func (s Scanner) Keywords() []string { // FromData will find and optionally verify Enigma secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) for _, match := range matches { @@ -46,20 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := strings.NewReader(`{"name":"Enigma Technologies, Inc.","person":{"first_name":"","last_name":""},"address":{"street_address1":"245 5th Ave","street_address2":"","city":"New York","state":"NY","postal_code":"10016"}}`) - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.enigma.com/businesses/match", payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("x-api-key", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + isVerified, verificationErr := verifyKey(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -68,10 +58,52 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +// verifyKey checks an Enigma API key against the GraphQL endpoint. +// +// The query selects only `__typename`, which the GraphQL spec guarantees on +// every root type, so verification does not depend on Enigma's evolving schema +// or any specific entity id. Authentication is driven by the HTTP status code: +// 200 means the key is valid, 401 mean it is not. +// +// A 200 is additionally guarded by a JSON Content-Type check: server could return +// a 200 HTML block page, which we must not mistake for a live key. +func verifyKey(ctx context.Context, client *http.Client, key string) (bool, error) { + payload := strings.NewReader(`{"query":"{ __typename }"}`) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.enigma.com/graphql", payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("x-api-key", key) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + if contentType := res.Header.Get("Content-Type"); !strings.Contains(contentType, "application/json") { + return false, fmt.Errorf("got HTTP 200 with unexpected Content-Type %q", contentType) + } + + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Enigma } func (s Scanner) Description() string { - return "Enigma is a data intelligence company that provides comprehensive data about businesses. Enigma API keys can be used to access and interact with this data." + return "Enigma is a business data intelligence platform. Enigma API keys grant access to comprehensive firmographic, legal, and identity datasets." } diff --git a/pkg/detectors/enigma/enigma_integration_test.go b/pkg/detectors/enigma/enigma_integration_test.go index c0fbb725d43a..576db484c742 100644 --- a/pkg/detectors/enigma/enigma_integration_test.go +++ b/pkg/detectors/enigma/enigma_integration_test.go @@ -19,7 +19,7 @@ import ( func TestEnigma_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } @@ -50,6 +50,9 @@ func TestEnigma_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_Enigma, Verified: true, + SecretParts: map[string]string{ + "key": secret, + }, }, }, wantErr: false, @@ -66,6 +69,9 @@ func TestEnigma_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_Enigma, Verified: false, + SecretParts: map[string]string{ + "key": inactiveSecret, + }, }, }, wantErr: false, diff --git a/pkg/detectors/envoyapikey/envoyapikey.go b/pkg/detectors/envoyapikey/envoyapikey.go index 5ef0d8203206..153c4f01135e 100644 --- a/pkg/detectors/envoyapikey/envoyapikey.go +++ b/pkg/detectors/envoyapikey/envoyapikey.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() body, _ := io.ReadAll(res.Body) // Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body. diff --git a/pkg/detectors/etherscan/etherscan.go b/pkg/detectors/etherscan/etherscan.go index e46d2d99f65a..b2fb2bab5098 100644 --- a/pkg/detectors/etherscan/etherscan.go +++ b/pkg/detectors/etherscan/etherscan.go @@ -47,23 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.etherscan.io/api?module=account&action=balance&address=0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae&tag=latest&apikey="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - if strings.Contains(body, `"OK"`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -72,6 +58,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.etherscan.io/api?module=account&action=balance&address=0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae&tag=latest&apikey="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + if strings.Contains(body, `"OK"`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Etherscan } diff --git a/pkg/detectors/ethplorer/ethplorer.go b/pkg/detectors/ethplorer/ethplorer.go index b113206a2996..08ce6327e51e 100644 --- a/pkg/detectors/ethplorer/ethplorer.go +++ b/pkg/detectors/ethplorer/ethplorer.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/everhour/everhour.go b/pkg/detectors/everhour/everhour.go index c04ddb966fc1..a2583ef0da3f 100644 --- a/pkg/detectors/everhour/everhour.go +++ b/pkg/detectors/everhour/everhour.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/exchangeratesapi/exchangeratesapi.go b/pkg/detectors/exchangeratesapi/exchangeratesapi.go index 20c600312c33..88f65b2d01cc 100644 --- a/pkg/detectors/exchangeratesapi/exchangeratesapi.go +++ b/pkg/detectors/exchangeratesapi/exchangeratesapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/exportsdk/exportsdk.go b/pkg/detectors/exportsdk/exportsdk.go index aaf81e8e52f3..81590080cfdc 100644 --- a/pkg/detectors/exportsdk/exportsdk.go +++ b/pkg/detectors/exportsdk/exportsdk.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-KEY", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/extractorapi/extractorapi.go b/pkg/detectors/extractorapi/extractorapi.go index 6bd17a1e2380..4ad6ee34c979 100644 --- a/pkg/detectors/extractorapi/extractorapi.go +++ b/pkg/detectors/extractorapi/extractorapi.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/facebookoauth/facebookoauth.go b/pkg/detectors/facebookoauth/facebookoauth.go index 1793b59be586..e81b0c7f7b0f 100644 --- a/pkg/detectors/facebookoauth/facebookoauth.go +++ b/pkg/detectors/facebookoauth/facebookoauth.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/faceplusplus/faceplusplus.go b/pkg/detectors/faceplusplus/faceplusplus.go index 4a771336aa3b..c9f1ea80591c 100644 --- a/pkg/detectors/faceplusplus/faceplusplus.go +++ b/pkg/detectors/faceplusplus/faceplusplus.go @@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fastforex/fastforex.go b/pkg/detectors/fastforex/fastforex.go index 88ebaf8aeeea..2d7271d4e252 100644 --- a/pkg/detectors/fastforex/fastforex.go +++ b/pkg/detectors/fastforex/fastforex.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go index 639fa85b1c75..0a90940c8566 100644 --- a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go +++ b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go @@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_FastlyPersonalToken, Raw: []byte(match), + SecretParts: map[string]string{"key": match}, } if verify { @@ -59,12 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = verified s1.ExtraData = extraData s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": match, - } - } } results = append(results, s1) diff --git a/pkg/detectors/feedier/feedier.go b/pkg/detectors/feedier/feedier.go index 3c066449fb4c..c1a5939ecf97 100644 --- a/pkg/detectors/feedier/feedier.go +++ b/pkg/detectors/feedier/feedier.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken.go b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken.go new file mode 100644 index 000000000000..6fe8583beeae --- /dev/null +++ b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken.go @@ -0,0 +1,70 @@ +package figmapersonalaccesstoken + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +// VerifyMatch checks if the provided Figma token is valid by making a request to the Figma API. +// +// Tokens are passed via the X-Figma-Token header: +// https://developers.figma.com/docs/rest-api/personal-access-tokens/ +// +// Figma documents 400, 403, 404, 429 and 500, and returns 403 for every +// authentication failure: https://developers.figma.com/docs/rest-api/errors/ +// We are keeping 401 here for completeness, but it is not documented. +func VerifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.figma.com/v1/me", http.NoBody) + if err != nil { + return false, err + } + + req.Header.Add("X-Figma-Token", token) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + case http.StatusForbidden: + return verify403(res.Body) + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +// verify403 distinguishes a dead token from a live one. /v1/me requires the +// current_user:read scope (https://developers.figma.com/docs/rest-api/scopes/), +// so a live token issued without it is rejected with the same 403 as a revoked +// one. The error message is the only thing that separates them. +func verify403(body io.Reader) (bool, error) { + var res struct { + Err string `json:"err"` + } + if err := json.NewDecoder(body).Decode(&res); err != nil { + return false, fmt.Errorf("decoding 403 response body: %w", err) + } + + switch { + case res.Err == "Invalid token": + return false, nil + case strings.HasPrefix(res.Err, "Invalid scope(s):"): + return true, nil + default: + // Reported as indeterminate rather than unverified so an unrecognised + // message surfaces instead of silently becoming a false negative. + return false, fmt.Errorf("unexpected 403 response: %q", res.Err) + } +} diff --git a/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go new file mode 100644 index 000000000000..5c1cf0b87c5f --- /dev/null +++ b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go @@ -0,0 +1,85 @@ +package figmapersonalaccesstoken + +import ( + "context" + "testing" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" +) + +func TestVerifyMatch(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantVerified bool + wantErr bool + }{ + { + name: "200 OK is verified", + statusCode: 200, + wantVerified: true, + wantErr: false, + }, + { + name: "403 invalid token is unverified without error", + statusCode: 403, + body: `{"status":403,"err":"Invalid token"}`, + wantVerified: false, + wantErr: false, + }, + { + name: "403 missing scope is a live token", + statusCode: 403, + body: `{"status":403,"err":"Invalid scope(s): file_comments:read. This endpoint requires the current_user:read scope"}`, + wantVerified: true, + wantErr: false, + }, + { + name: "403 with an unrecognised message is indeterminate", + statusCode: 403, + body: `{"status":403,"err":"Request denied"}`, + wantVerified: false, + wantErr: true, + }, + { + name: "403 with an unparseable body is indeterminate", + statusCode: 403, + body: "not json", + wantVerified: false, + wantErr: true, + }, + { + name: "401 Unauthorized is unverified with no error", + statusCode: 401, + wantVerified: false, + wantErr: false, + }, + { + name: "429 rate limited is unverified with error", + statusCode: 429, + wantVerified: false, + wantErr: true, + }, + { + name: "500 Internal Server Error is unverified with error", + statusCode: 500, + wantVerified: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := common.ConstantResponseHttpClient(tt.statusCode, tt.body) + verified, err := VerifyMatch(context.Background(), client, "figp_test_token") + + if verified != tt.wantVerified { + t.Errorf("VerifyMatch() verified = %v, want %v", verified, tt.wantVerified) + } + if (err != nil) != tt.wantErr { + t.Errorf("VerifyMatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalaccesstoken.go b/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalaccesstoken.go index 4a6558eab3fa..be0e36a0d13e 100644 --- a/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalaccesstoken.go +++ b/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalaccesstoken.go @@ -10,6 +10,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + figma "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/figmapersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -25,8 +26,7 @@ func (Scanner) Version() int { return 1 } var ( defaultClient = common.SaneHttpClient() - - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"figma"}) + `\b([0-9]{6}-[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"figma"}) + `\b([0-9]{6}-[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -35,6 +35,17 @@ func (s Scanner) Keywords() []string { return []string{"figma"} } +func (s Scanner) Description() string { + return "Figma is a web-based design tool. Personal Access Tokens can be used to access and modify design files and other resources." +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify FigmaPersonalAccessToken secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -45,39 +56,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result resMatch := strings.TrimSpace(match[1]) s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_FigmaPersonalAccessToken, + DetectorType: s.Type(), Raw: []byte(resMatch), ExtraData: map[string]string{ - "version": fmt.Sprintf("%d", s.Version()), + "version": fmt.Sprint(s.Version()), }, + SecretParts: map[string]string{"token": resMatch}, } if verify { - client := s.client - if client == nil { - client = defaultClient - } - - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.figma.com/v1/me", nil) - if err != nil { - continue - } - req.Header.Add("X-Figma-Token", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else if res.StatusCode != 403 { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resMatch) - } - } else { - s1.SetVerificationError(err, resMatch) - } - if s1.Verified { - s1.SecretParts = map[string]string{"token": resMatch} - } + isVerified, verificationErr := figma.VerifyMatch(ctx, s.getClient(), resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -89,7 +79,3 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_FigmaPersonalAccessToken } - -func (s Scanner) Description() string { - return "Figma is a web-based design tool. Personal Access Tokens can be used to access and modify design files and other resources." -} diff --git a/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalacesstoken_integration_test.go b/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalacesstoken_integration_test.go index a05e63fbc65b..bc5c84fd5765 100644 --- a/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalacesstoken_integration_test.go +++ b/pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalacesstoken_integration_test.go @@ -145,7 +145,7 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("FigmaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_integration_test.go b/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_integration_test.go index d9da51d71e42..fb33efe4b8e1 100644 --- a/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_integration_test.go +++ b/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_integration_test.go @@ -145,7 +145,7 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("FigmaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_v2.go b/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_v2.go index 061e82e815bc..af93b044021d 100644 --- a/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_v2.go +++ b/pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_v2.go @@ -10,6 +10,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + figma "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/figmapersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -39,6 +40,13 @@ func (s Scanner) Description() string { return "Figma is a collaborative interface design tool. Figma Personal Access Tokens can be used to access and manipulate design files and other resources on behalf of a user." } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify FigmaPersonalAccessToken secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -49,39 +57,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result resMatch := strings.TrimSpace(match[1]) s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_FigmaPersonalAccessToken, + DetectorType: s.Type(), Raw: []byte(resMatch), ExtraData: map[string]string{ - "version": fmt.Sprintf("%d", s.Version()), + "version": fmt.Sprint(s.Version()), }, + SecretParts: map[string]string{"token": resMatch}, } if verify { - client := s.client - if client == nil { - client = defaultClient - } - - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.figma.com/v1/me", nil) - if err != nil { - continue - } - req.Header.Add("X-Figma-Token", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else if res.StatusCode != 403 { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resMatch) - } - } else { - s1.SetVerificationError(err, resMatch) - } - if s1.Verified { - s1.SecretParts = map[string]string{"token": resMatch} - } + isVerified, verificationErr := figma.VerifyMatch(ctx, s.getClient(), resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) diff --git a/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken.go b/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken.go new file mode 100644 index 000000000000..642847e26857 --- /dev/null +++ b/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken.go @@ -0,0 +1,83 @@ +package figmapersonalaccesstoken + +import ( + "context" + "fmt" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + figma "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/figmapersonalaccesstoken" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 3 } + +var ( + defaultClient = common.SaneHttpClient() + // The figp_ token charset includes '=' and '-', which are not word characters, + // so a \b word boundary can't be used to anchor the end of the match. Instead we + // require the next character to be outside the token charset (or end of input). + keyPat = regexp.MustCompile(`(figp_[a-zA-Z0-9_=-]{40,54})(?:[^a-zA-Z0-9_=-]|\z)`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"figp_"} +} + +func (s Scanner) Description() string { + return "Figma is a collaborative interface design tool. Figma Personal Access Tokens can be used to access and manipulate design files and other resources on behalf of a user." +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData will find and optionally verify FigmaPersonalAccessToken secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + ExtraData: map[string]string{ + "version": fmt.Sprint(s.Version()), + }, + SecretParts: map[string]string{"token": resMatch}, + } + + if verify { + isVerified, verificationErr := figma.VerifyMatch(ctx, s.getClient(), resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + results = append(results, s1) + } + + return results, nil +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_FigmaPersonalAccessToken +} diff --git a/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken_integration_test.go b/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken_integration_test.go new file mode 100644 index 000000000000..1c9f81db71c7 --- /dev/null +++ b/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken_integration_test.go @@ -0,0 +1,171 @@ +//go:build detectors +// +build detectors + +package figmapersonalaccesstoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) { + t.Skip("skipping until v3 secrets are provisioned in GCP") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("FIGMAPERSONALACCESSTOKEN_V3_TOKEN") + inactiveSecret := testSecrets.MustGetField("FIGMAPERSONALACCESSTOKEN_V3_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_FigmaPersonalAccessToken, + Verified: true, + ExtraData: map[string]string{ + "version": "3", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within but not valid", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_FigmaPersonalAccessToken, + Verified: false, + ExtraData: map[string]string{ + "version": "3", + }, + }, + }, + wantErr: false, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_FigmaPersonalAccessToken, + Verified: false, + ExtraData: map[string]string{ + "version": "3", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a figmapersonalaccesstoken secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_FigmaPersonalAccessToken, + Verified: false, + ExtraData: map[string]string{ + "version": "3", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("FigmaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("FigmaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken_test.go b/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken_test.go new file mode 100644 index 000000000000..534a87d4264e --- /dev/null +++ b/pkg/detectors/figmapersonalaccesstoken/v3/figmapersonalaccesstoken_test.go @@ -0,0 +1,122 @@ +package figmapersonalaccesstoken + +import ( + "context" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = `[{ + "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5", + "name": "FigmaV3", + "type": "Detector", + "api": true, + "authentication_type": "", + "verification_url": "https://api.figma.com/v1/me", + "test_secrets": { + "figma_secret": "figp_EZe7plhYvN92IyiDCjkvTcbNVZsuRVpDcHOwNNP1" + }, + "expected_response": "200", + "method": "GET", + "deprecated": false + }]` + secret = "figp_EZe7plhYvN92IyiDCjkvTcbNVZsuRVpDcHOwNNP1" + + // A token ending in '=' or '-' can't be anchored with \b, so keyPat requires the + // next character to fall outside the token charset instead. + secretWithTrailingPadding = "figp_EZe7plhYvN92IyiDCjkvTcbNVZsuRVpDcHOwNN==" + + invalidPatterns = []string{ + `figp_short`, // too short + `figd_EZe7plhYvN92IyiDCjkvTcbNVZsuRVpDcHOwNNP1`, // wrong prefix (v2 token) + `figp_!!!invalid_chars_here!!!!!!!!!!!!!!!!!!!!!!!!!`, // invalid characters + `figp_` + strings.Repeat("a", 60), // over-length, must not match a truncated prefix + } +) + +func TestFigmaPersonalAccessToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: validPattern, + want: []string{secret}, + }, + { + name: "token ending in padding characters", + input: `FigmaAuthToken=` + secretWithTrailingPadding, + want: []string{secretWithTrailingPadding}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + +func TestFigmaPersonalAccessToken_InvalidPatterns(t *testing.T) { + d := Scanner{} + + for _, input := range invalidPatterns { + t.Run(input, func(t *testing.T) { + results, err := d.FromData(context.Background(), false, []byte(input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + if len(results) != 0 { + t.Errorf("expected no results for invalid input %q, got %d", input, len(results)) + } + }) + } +} diff --git a/pkg/detectors/fileio/fileio.go b/pkg/detectors/fileio/fileio.go index 8b6ec9c86443..4dcb0e887045 100644 --- a/pkg/detectors/fileio/fileio.go +++ b/pkg/detectors/fileio/fileio.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := io.ReadAll(res.Body) if err == nil { isJson := json.Valid(bodyBytes) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if isJson { s1.Verified = true diff --git a/pkg/detectors/finage/finage.go b/pkg/detectors/finage/finage.go index 27b5a1038b52..a4251d0d468b 100644 --- a/pkg/detectors/finage/finage.go +++ b/pkg/detectors/finage/finage.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Finage, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/financialmodelingprep/financialmodelingprep.go b/pkg/detectors/financialmodelingprep/financialmodelingprep.go index e0309dcd9885..b350d545993a 100644 --- a/pkg/detectors/financialmodelingprep/financialmodelingprep.go +++ b/pkg/detectors/financialmodelingprep/financialmodelingprep.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // error response is in json validResponse := strings.Contains(bodyString, `[ "`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/findl/findl.go b/pkg/detectors/findl/findl.go index 657b5e09fd46..c77f6085b8af 100644 --- a/pkg/detectors/findl/findl.go +++ b/pkg/detectors/findl/findl.go @@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/finnhub/finnhub.go b/pkg/detectors/finnhub/finnhub.go index a8014d7dbd3f..85c531d5083e 100644 --- a/pkg/detectors/finnhub/finnhub.go +++ b/pkg/detectors/finnhub/finnhub.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fixerio/fixerio.go b/pkg/detectors/fixerio/fixerio.go index d499c6d0ab4f..8999e344a6e1 100644 --- a/pkg/detectors/fixerio/fixerio.go +++ b/pkg/detectors/fixerio/fixerio.go @@ -47,32 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://data.fixer.io/api/latest?access_key="+resMatch, nil) - if err != nil { - continue - } - - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} - // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} - // ingenious! - - validResponse := strings.Contains(body, `"success": true`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() - - if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { - s1.Verified = true - } - - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -81,6 +58,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://data.fixer.io/api/latest?access_key="+resMatch, nil) + if err != nil { + return false, err + } + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} + // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} + // ingenious! + + validResponse := strings.Contains(body, `"success": true`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) + + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_FixerIO } diff --git a/pkg/detectors/flatio/flatio.go b/pkg/detectors/flatio/flatio.go index 7af1ce3659ab..2d215d758996 100644 --- a/pkg/detectors/flatio/flatio.go +++ b/pkg/detectors/flatio/flatio.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fleetbase/fleetbase.go b/pkg/detectors/fleetbase/fleetbase.go index a00efc3d1620..12c03523056e 100644 --- a/pkg/detectors/fleetbase/fleetbase.go +++ b/pkg/detectors/fleetbase/fleetbase.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/flickr/flickr.go b/pkg/detectors/flickr/flickr.go index f01c0089225c..e67e3ee8f88f 100644 --- a/pkg/detectors/flickr/flickr.go +++ b/pkg/detectors/flickr/flickr.go @@ -48,23 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.flickr.com/services/rest/?method=flickr.tags.getHotList&api_key=%s", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "owner=") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -73,6 +59,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.flickr.com/services/rest/?method=flickr.tags.getHotList&api_key=%s", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "owner=") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Flickr } diff --git a/pkg/detectors/flightapi/flightapi.go b/pkg/detectors/flightapi/flightapi.go index 2d480caf793c..67d1fc6b667d 100644 --- a/pkg/detectors/flightapi/flightapi.go +++ b/pkg/detectors/flightapi/flightapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/flightstats/flightstats.go b/pkg/detectors/flightstats/flightstats.go index b2aace0b2f8c..32ed5d12acbd 100644 --- a/pkg/detectors/flightstats/flightstats.go +++ b/pkg/detectors/flightstats/flightstats.go @@ -55,24 +55,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.flightstats.com/flex/aircraft/rest/v1/json/availableFields?appId=%s&appKey=%s", resId, resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - validResponse := (res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "id")) || (res.StatusCode == 403 && strings.Contains(body, "application is not active")) - if validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resId, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resId, resMatch) } results = append(results, s1) @@ -82,6 +67,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resId string, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.flightstats.com/flex/aircraft/rest/v1/json/availableFields?appId=%s&appKey=%s", resId, resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + validResponse := (res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "id")) || (res.StatusCode == 403 && strings.Contains(body, "application is not active")) + if validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Flightstats } diff --git a/pkg/detectors/float/float.go b/pkg/detectors/float/float.go index 186825f6c297..28da924c49bd 100644 --- a/pkg/detectors/float/float.go +++ b/pkg/detectors/float/float.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("User-Agent", "TruffleHog3 (example@example.com)") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/flowflu/flowflu.go b/pkg/detectors/flowflu/flowflu.go index 95acf15e3bd5..29fc83b0bfb0 100644 --- a/pkg/detectors/flowflu/flowflu.go +++ b/pkg/detectors/flowflu/flowflu.go @@ -56,29 +56,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.flowlu.com/api/v1/module/crm/lead/list?api_key=%s", resAccount, resMatch), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `total_result`) - - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resAccount, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resAccount, resMatch) } results = append(results, s1) @@ -88,6 +68,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resAccount string, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.flowlu.com/api/v1/module/crm/lead/list?api_key=%s", resAccount, resMatch), nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `total_result`) + + if res.StatusCode >= 200 && res.StatusCode < 300 { + if validResponse { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_FlowFlu } diff --git a/pkg/detectors/flutterwave/flutterwave.go b/pkg/detectors/flutterwave/flutterwave.go index 5f5bf3d342ca..19a8cf7987f9 100644 --- a/pkg/detectors/flutterwave/flutterwave.go +++ b/pkg/detectors/flutterwave/flutterwave.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fmfw/fmfw.go b/pkg/detectors/fmfw/fmfw.go index 92266ba39a8a..7b85d371fb1f 100644 --- a/pkg/detectors/fmfw/fmfw.go +++ b/pkg/detectors/fmfw/fmfw.go @@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/formbucket/formbucket.go b/pkg/detectors/formbucket/formbucket.go index 3f7cd894282a..76441a2cea99 100644 --- a/pkg/detectors/formbucket/formbucket.go +++ b/pkg/detectors/formbucket/formbucket.go @@ -56,14 +56,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() body, errBody := io.ReadAll(res.Body) if errBody != nil { continue } bodyString := string(body) validResponse := strings.Contains(bodyString, `created_on`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if errBody == nil { if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { s1.Verified = true diff --git a/pkg/detectors/formcraft/formcraft.go b/pkg/detectors/formcraft/formcraft.go index 3590fa0ac719..b1c9754abe7f 100644 --- a/pkg/detectors/formcraft/formcraft.go +++ b/pkg/detectors/formcraft/formcraft.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/formio/formio.go b/pkg/detectors/formio/formio.go index 53f3088a30af..3c13423c2af7 100644 --- a/pkg/detectors/formio/formio.go +++ b/pkg/detectors/formio/formio.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-jwt-token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/formsite/formsite.go b/pkg/detectors/formsite/formsite.go index 2373d4839c6a..dc0e19e75964 100644 --- a/pkg/detectors/formsite/formsite.go +++ b/pkg/detectors/formsite/formsite.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/foursquare/foursquare.go b/pkg/detectors/foursquare/foursquare.go index c42a862cdb98..979bbd63a743 100644 --- a/pkg/detectors/foursquare/foursquare.go +++ b/pkg/detectors/foursquare/foursquare.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/frameio/frameio.go b/pkg/detectors/frameio/frameio.go index a6f869019604..52e23dae6a0b 100644 --- a/pkg/detectors/frameio/frameio.go +++ b/pkg/detectors/frameio/frameio.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go index d3c2385f2fad..893de60d1a28 100644 --- a/pkg/detectors/freshbooks/freshbooks.go +++ b/pkg/detectors/freshbooks/freshbooks.go @@ -53,22 +53,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(`https://auth.freshbooks.com/oauth/authorize?client_id=%s&redirect_uri=%s&response_type=code`, resMatch, resURI), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "Log In to FreshBooks") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch, resURI) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch, resURI) } results = append(results, s1) @@ -78,6 +65,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string, resURI string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(`https://auth.freshbooks.com/oauth/authorize?client_id=%s&redirect_uri=%s&response_type=code`, resMatch, resURI), nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "Log In to FreshBooks") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Freshbooks } diff --git a/pkg/detectors/freshdesk/freshdesk.go b/pkg/detectors/freshdesk/freshdesk.go index 0631aeda41a7..9b4a66ee6cca 100644 --- a/pkg/detectors/freshdesk/freshdesk.go +++ b/pkg/detectors/freshdesk/freshdesk.go @@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/front/front.go b/pkg/detectors/front/front.go index e6a50b0f9e84..060c221df2c4 100644 --- a/pkg/detectors/front/front.go +++ b/pkg/detectors/front/front.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.go index 16ad41dcce73..db757b35e6b8 100644 --- a/pkg/detectors/ftp/ftp.go +++ b/pkg/detectors/ftp/ftp.go @@ -70,7 +70,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result rawURL, _ := url.Parse(urlMatch) rawURL.Path = "" - redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1)) + redact := strings.TrimSpace(strings.ReplaceAll(rawURL.String(), password, "********")) s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_FTP, @@ -139,7 +139,7 @@ func verifyFTP(timeout time.Duration, u *url.URL) error { return nil, err } if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { - conn.Close() + _ = conn.Close() return nil, err } return conn, nil diff --git a/pkg/detectors/fulcrum/fulcrum.go b/pkg/detectors/fulcrum/fulcrum.go index 57113c6054b9..065eb7ef7493 100644 --- a/pkg/detectors/fulcrum/fulcrum.go +++ b/pkg/detectors/fulcrum/fulcrum.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-ApiToken", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fullstory/v1/fullstory.go b/pkg/detectors/fullstory/v1/fullstory.go index 15cdce5eaa26..e5de71e78a44 100644 --- a/pkg/detectors/fullstory/v1/fullstory.go +++ b/pkg/detectors/fullstory/v1/fullstory.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fullstory/v2/fullstory_v2.go b/pkg/detectors/fullstory/v2/fullstory_v2.go index 1918ad6a309d..aa719c70acdd 100644 --- a/pkg/detectors/fullstory/v2/fullstory_v2.go +++ b/pkg/detectors/fullstory/v2/fullstory_v2.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/fxmarket/fxmarket.go b/pkg/detectors/fxmarket/fxmarket.go index cbadf5bc4c85..f0e824cb25f1 100644 --- a/pkg/detectors/fxmarket/fxmarket.go +++ b/pkg/detectors/fxmarket/fxmarket.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.go index dc911d80b5ac..e7b8ee2f992d 100644 --- a/pkg/detectors/gcp/gcp.go +++ b/pkg/detectors/gcp/gcp.go @@ -212,7 +212,7 @@ func findMatchingCertificateKID(ctx context.Context, certsURL, privateKeyPEM str } // Compare RSA public keys - if publicKey.PublicKey.N.Cmp(certPublicKey.N) == 0 && publicKey.PublicKey.E == certPublicKey.E { + if publicKey.N.Cmp(certPublicKey.N) == 0 && publicKey.E == certPublicKey.E { return kid, nil } } @@ -233,7 +233,7 @@ func fetchServiceAccountCerts(ctx context.Context, certsURL string) (map[string] if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, nil diff --git a/pkg/detectors/geckoboard/geckoboard.go b/pkg/detectors/geckoboard/geckoboard.go index 4bd76c802eff..46e81f25feec 100644 --- a/pkg/detectors/geckoboard/geckoboard.go +++ b/pkg/detectors/geckoboard/geckoboard.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gemini/gemini.go b/pkg/detectors/gemini/gemini.go index 283723d5bbea..5fe1eac66adc 100644 --- a/pkg/detectors/gemini/gemini.go +++ b/pkg/detectors/gemini/gemini.go @@ -77,7 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gengo/gengo.go b/pkg/detectors/gengo/gengo.go index d1aa53b5c9fe..9178bfe7cfbd 100644 --- a/pkg/detectors/gengo/gengo.go +++ b/pkg/detectors/gengo/gengo.go @@ -66,31 +66,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - - timestamp := strconv.FormatInt(time.Now().Unix(), 10) - signature := getGengoSignature(timestamp, resSecretMatch) - - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.gengo.com/v2/account/me?ts=%s&api_key=%s&api_sig=%s", timestamp, resMatch, signature), nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - body, errBody := io.ReadAll(res.Body) - - if errBody == nil { - var response Response - if err := json.Unmarshal(body, &response); err != nil { - continue - } - - if res.StatusCode >= 200 && res.StatusCode < 300 && response.OpStat == "ok" { - s1.Verified = true - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resSecretMatch, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resSecretMatch, resMatch) } results = append(results, s1) @@ -100,6 +78,38 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resSecretMatch string, resMatch string) (bool, error) { + + timestamp := strconv.FormatInt(time.Now().Unix(), 10) + signature := getGengoSignature(timestamp, resSecretMatch) + + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.gengo.com/v2/account/me?ts=%s&api_key=%s&api_sig=%s", timestamp, resMatch, signature), nil) + if err != nil { + return false, err + } + req.Header.Add("Accept", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + body, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + var response Response + if err := json.Unmarshal(body, &response); err != nil { + return false, err + } + + if res.StatusCode >= 200 && res.StatusCode < 300 && response.OpStat == "ok" { + return true, nil + } + + return false, nil +} + type Response struct { OpStat string `json:"opstat"` } diff --git a/pkg/detectors/geoapify/geoapify.go b/pkg/detectors/geoapify/geoapify.go index 09d6739e0b34..956cd90902bc 100644 --- a/pkg/detectors/geoapify/geoapify.go +++ b/pkg/detectors/geoapify/geoapify.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/geocode/geocode.go b/pkg/detectors/geocode/geocode.go index 265d595d51f8..c1a5a030b84e 100644 --- a/pkg/detectors/geocode/geocode.go +++ b/pkg/detectors/geocode/geocode.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"remaining_credits" :`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/geocodify/geocodify.go b/pkg/detectors/geocodify/geocodify.go index 9a592a76f36a..db3538895f94 100644 --- a/pkg/detectors/geocodify/geocodify.go +++ b/pkg/detectors/geocodify/geocodify.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/geocodio/geocodio.go b/pkg/detectors/geocodio/geocodio.go index c9ced6834f1f..157d0b4888d7 100644 --- a/pkg/detectors/geocodio/geocodio.go +++ b/pkg/detectors/geocodio/geocodio.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/geoipifi/geoipifi.go b/pkg/detectors/geoipifi/geoipifi.go index 7547362aea1e..b03f6e132723 100644 --- a/pkg/detectors/geoipifi/geoipifi.go +++ b/pkg/detectors/geoipifi/geoipifi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/getemail/getemail.go b/pkg/detectors/getemail/getemail.go index deab794a3e62..0aed3b29fb48 100644 --- a/pkg/detectors/getemail/getemail.go +++ b/pkg/detectors/getemail/getemail.go @@ -49,27 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.getemail.io/v1/find-mail?firstname=Larry&lastname=Page&domain=google.com&api_key=%s", resMatch), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":"USER_NOT_EXIST"`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { - s1.Verified = true - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -78,6 +60,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.getemail.io/v1/find-mail?firstname=Larry&lastname=Page&domain=google.com&api_key=%s", resMatch), nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(bodyBytes) + errCode := strings.Contains(bodyString, `"code":"USER_NOT_EXIST"`) + if res.StatusCode >= 200 && res.StatusCode < 300 { + if !errCode { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_GetEmail } diff --git a/pkg/detectors/getemails/getemails.go b/pkg/detectors/getemails/getemails.go index a509ff0f25ea..e049b45e2039 100644 --- a/pkg/detectors/getemails/getemails.go +++ b/pkg/detectors/getemails/getemails.go @@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/getgeoapi/getgeoapi.go b/pkg/detectors/getgeoapi/getgeoapi.go index bff3e7c368de..2a723b60f99c 100644 --- a/pkg/detectors/getgeoapi/getgeoapi.go +++ b/pkg/detectors/getgeoapi/getgeoapi.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/getgist/getgist.go b/pkg/detectors/getgist/getgist.go index 87fbb1343a1c..2b88b53ad54b 100644 --- a/pkg/detectors/getgist/getgist.go +++ b/pkg/detectors/getgist/getgist.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/getresponse/getresponse.go b/pkg/detectors/getresponse/getresponse.go index 7b02af9414d8..c89a7a21e35a 100644 --- a/pkg/detectors/getresponse/getresponse.go +++ b/pkg/detectors/getresponse/getresponse.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Auth-Token", fmt.Sprintf("api-key %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/getsandbox/getsandbox.go b/pkg/detectors/getsandbox/getsandbox.go index 248be740f415..67157a530c39 100644 --- a/pkg/detectors/getsandbox/getsandbox.go +++ b/pkg/detectors/getsandbox/getsandbox.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("API-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.go index f907b36f37c6..393c16553533 100644 --- a/pkg/detectors/github/v1/github_old.go +++ b/pkg/detectors/github/v1/github_old.go @@ -170,7 +170,7 @@ func (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token st if res.StatusCode >= 200 && res.StatusCode < 300 { var userResponse UserRes err = json.NewDecoder(res.Body).Decode(&userResponse) - res.Body.Close() + _ = res.Body.Close() if err == nil { // GitHub does not seem to consistently return this header. scopes := res.Header.Get("X-OAuth-Scopes") diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.go index bd3581237737..e20ef6c28558 100644 --- a/pkg/detectors/github/v2/github.go +++ b/pkg/detectors/github/v2/github.go @@ -3,6 +3,7 @@ package github import ( "context" "fmt" + "strings" regexp "github.com/wasilibs/go-re2" @@ -37,8 +38,34 @@ var ( // TODO: Oauth2 client_id and client_secret // https://developer.github.com/v3/#oauth2-keysecret + + // tokenTypesByPrefix maps each GitHub token prefix to a human-readable type. + // Each prefix identifies a materially different credential (PAT, OAuth + // grant, or GitHub App token) that is revoked/rotated through a different + // GitHub settings page. + // https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ + tokenTypesByPrefix = map[string]string{ + "ghp_": "Personal Access Token (classic)", + "github_pat_": "Personal Access Token (fine-grained)", + "gho_": "OAuth Access Token", + "ghu_": "GitHub App User-to-Server Token", + "ghs_": "GitHub App Server-to-Server (installation) Token", + "ghr_": "GitHub App Refresh Token", + } ) +// githubTokenType returns the human-readable token type for a matched token, +// keyed off its prefix. Falls back to a generic label if no known prefix +// matches (should not happen given keyPat, but keeps this safe). +func githubTokenType(token string) string { + for prefix, tokenType := range tokenTypesByPrefix { + if strings.HasPrefix(token, prefix) { + return tokenType + } + } + return "Unknown GitHub token" +} + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { @@ -62,6 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", "version": fmt.Sprintf("%d", s.Version()), + "token_type": githubTokenType(token), }, SecretParts: map[string]string{"key": token}, } diff --git a/pkg/detectors/github/v2/github_integration_test.go b/pkg/detectors/github/v2/github_integration_test.go index 61861f5690f6..20715b6a719b 100644 --- a/pkg/detectors/github/v2/github_integration_test.go +++ b/pkg/detectors/github/v2/github_integration_test.go @@ -62,9 +62,10 @@ func TestGitHub_FromChunk(t *testing.T) { "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", "scopes": "notifications", // "site_admin": "false", // not present in test verifiedGhp - "url": "https://github.com/truffle-sandbox", - "username": "truffle-sandbox", - "version": "2", + "token_type": "Personal Access Token (classic)", + "url": "https://github.com/truffle-sandbox", + "username": "truffle-sandbox", + "version": "2", }, }, }, @@ -84,6 +85,7 @@ func TestGitHub_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", + "token_type": "Personal Access Token (classic)", "version": "2", }, }, @@ -104,6 +106,7 @@ func TestGitHub_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", + "token_type": "OAuth Access Token", "version": "2", }, }, @@ -124,6 +127,7 @@ func TestGitHub_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", + "token_type": "GitHub App User-to-Server Token", "version": "2", }, }, @@ -144,6 +148,7 @@ func TestGitHub_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", + "token_type": "GitHub App Server-to-Server (installation) Token", "version": "2", }, }, @@ -164,6 +169,7 @@ func TestGitHub_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", + "token_type": "GitHub App Refresh Token", "version": "2", }, }, @@ -184,6 +190,7 @@ func TestGitHub_FromChunk(t *testing.T) { Verified: false, ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", + "token_type": "Personal Access Token (classic)", "version": "2", }, }, diff --git a/pkg/detectors/github/v2/github_test.go b/pkg/detectors/github/v2/github_test.go index cc769e82a22a..263eed8329eb 100644 --- a/pkg/detectors/github/v2/github_test.go +++ b/pkg/detectors/github/v2/github_test.go @@ -2,8 +2,12 @@ package github import ( "context" + "reflect" + "strings" "testing" + regexp "github.com/wasilibs/go-re2" + "github.com/google/go-cmp/cmp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -86,3 +90,175 @@ func TestGithub_Pattern(t *testing.T) { }) } } + +func TestGithubTokenType(t *testing.T) { + tests := []struct { + prefix string + token string + want string + }{ + {"ghp_", "ghp_" + strings.Repeat("a", 36), "Personal Access Token (classic)"}, + {"github_pat_", "github_pat_" + strings.Repeat("a", 36), "Personal Access Token (fine-grained)"}, + {"gho_", "gho_" + strings.Repeat("a", 36), "OAuth Access Token"}, + {"ghu_", "ghu_" + strings.Repeat("a", 36), "GitHub App User-to-Server Token"}, + {"ghs_", "ghs_" + strings.Repeat("a", 36), "GitHub App Server-to-Server (installation) Token"}, + {"ghr_", "ghr_" + strings.Repeat("a", 36), "GitHub App Refresh Token"}, + {"unknown", "not_a_github_token", "Unknown GitHub token"}, + } + + for _, test := range tests { + t.Run(test.prefix, func(t *testing.T) { + got := githubTokenType(test.token) + if got != test.want { + t.Errorf("githubTokenType(%q) = %q, want %q", test.token, got, test.want) + } + }) + } +} + +// prefixAlternationPat pulls the prefix alternation out of keyPat's source, +// e.g. `(?:ghp|gho|ghu|ghs|ghr|github_pat)_` -> `ghp|gho|ghu|ghs|ghr|github_pat`. +var prefixAlternationPat = regexp.MustCompile(`\(\?:([a-zA-Z0-9_|]+)\)_`) + +// keyPatPrefixes derives the token prefixes directly from keyPat's own source +// rather than from a hand-maintained list. If the two disagree, the regex is +// the authority — it is what actually decides whether a token is matched at +// all. +func keyPatPrefixes(t *testing.T) []string { + t.Helper() + + m := prefixAlternationPat.FindStringSubmatch(keyPat.String()) + if m == nil { + t.Fatalf("could not find a prefix alternation group in keyPat: %s\n"+ + "keyPat's shape changed; update prefixAlternationPat to match, or this "+ + "drift guard is silently testing nothing.", keyPat.String()) + } + + var prefixes []string + for _, alt := range strings.Split(m[1], "|") { + prefixes = append(prefixes, alt+"_") + } + return prefixes +} + +// TestGithubTokenType_MappingMatchesKeyPatExactly is the drift guard: it +// derives the prefix set from keyPat itself rather than a hand-maintained +// list, so a seventh prefix added to keyPat and forgotten in the map cannot +// slip past silently — the regex and the map cannot diverge without a +// failure. +// +// This matters more than a normal mapping test: an unmapped prefix does not +// crash, it silently reports "Unknown GitHub token", which puts a responder +// back in the position this detector change exists to fix — holding a real +// leaked credential with no idea how to revoke it. +func TestGithubTokenType_MappingMatchesKeyPatExactly(t *testing.T) { + prefixes := keyPatPrefixes(t) + + if len(prefixes) == 0 { + t.Fatal("derived zero prefixes from keyPat; the guard is not testing anything") + } + + fromKeyPat := make(map[string]bool, len(prefixes)) + for _, p := range prefixes { + fromKeyPat[p] = true + if _, ok := tokenTypesByPrefix[p]; !ok { + t.Errorf("keyPat matches prefix %q but tokenTypesByPrefix has no entry for it; "+ + "a leaked token with this prefix would be reported as %q", + p, "Unknown GitHub token") + } + } + + // The reverse direction: a mapping entry for a prefix keyPat cannot match is + // dead weight, and usually means a prefix was renamed in one place only. + for p := range tokenTypesByPrefix { + if !fromKeyPat[p] { + t.Errorf("tokenTypesByPrefix has an entry for %q, but keyPat never matches that prefix", p) + } + } +} + +// assertNoBlankFields checks that v is not a blank string, or — if v is a +// struct — that none of its exported fields are blank strings. It exists so +// that if a row's value type grows a second field (tokenTypesByPrefix held a +// {TokenType, Remediation} struct before remediation was removed, and could +// again), row validation below catches an unset field automatically instead +// of depending on someone remembering to add a field-specific check. +func assertNoBlankFields(t *testing.T, label string, v reflect.Value) { + t.Helper() + + switch v.Kind() { + case reflect.String: + if strings.TrimSpace(v.String()) == "" { + t.Errorf("%s: blank value", label) + } + case reflect.Struct: + vt := v.Type() + for i := 0; i < v.NumField(); i++ { + field := vt.Field(i) + if !field.IsExported() { + continue + } + assertNoBlankFields(t, label+"."+field.Name, v.Field(i)) + } + default: + t.Fatalf("%s: assertNoBlankFields does not know how to validate kind %s; extend the helper for the new row type", label, v.Kind()) + } +} + +// TestTokenTypesByPrefix_RowsAreWellFormed checks each row in +// tokenTypesByPrefix in isolation, independent of keyPat. The tests above +// confirm the map's keys line up with keyPat; none of them would catch a +// malformed row added alongside a correct key — e.g. a copy-pasted value +// that leaves two different prefixes describing the same token type, or a +// field left blank. +func TestTokenTypesByPrefix_RowsAreWellFormed(t *testing.T) { + const unknown = "Unknown GitHub token" + + seenValues := make(map[string]string, len(tokenTypesByPrefix)) // value -> first prefix that used it + + for prefix, tokenType := range tokenTypesByPrefix { + if prefix == "" { + t.Error("tokenTypesByPrefix has an empty-string prefix key") + } + if !strings.HasSuffix(prefix, "_") { + t.Errorf("prefix %q does not end in \"_\"; githubTokenType matches by strings.HasPrefix, so a bare prefix can match unrelated tokens", prefix) + } + assertNoBlankFields(t, prefix, reflect.ValueOf(tokenType)) + if tokenType == unknown { + t.Errorf("prefix %q maps to the reserved fallback value %q; a real row must not collide with it", prefix, unknown) + } + if other, dup := seenValues[tokenType]; dup { + t.Errorf("prefixes %q and %q both map to token type %q; each prefix must describe a distinct credential", prefix, other, tokenType) + } else { + seenValues[tokenType] = prefix + } + } +} + +// TestGithubTokenType_EveryKeyPatPrefixResolves closes the loop end to end: +// build a token for each prefix keyPat actually accepts, confirm keyPat +// matches it, and confirm githubTokenType resolves it to something other than +// the unknown fallback. The two tests above check the data structures agree; +// this one checks the behaviour they produce is correct for every real input. +func TestGithubTokenType_EveryKeyPatPrefixResolves(t *testing.T) { + const unknown = "Unknown GitHub token" + + for _, prefix := range keyPatPrefixes(t) { + t.Run(prefix, func(t *testing.T) { + token := prefix + strings.Repeat("a", 36) + + if !keyPat.MatchString(token) { + t.Fatalf("keyPat does not match %q built from its own prefix alternation; "+ + "the derived prefix is wrong or keyPat's body changed", token) + } + + got := githubTokenType(token) + if got == unknown { + t.Errorf("githubTokenType(%q) = %q; every prefix keyPat matches must resolve to a real type", token, got) + } + if got == "" { + t.Errorf("githubTokenType(%q) returned an empty type", token) + } + }) + } +} diff --git a/pkg/detectors/github_oauth2/github_oauth2.go b/pkg/detectors/github_oauth2/github_oauth2.go index a62376b2df83..8f18d4a284d7 100644 --- a/pkg/detectors/github_oauth2/github_oauth2.go +++ b/pkg/detectors/github_oauth2/github_oauth2.go @@ -19,10 +19,20 @@ type Scanner struct { // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) +// This detector emits the cartesian product of every client ID and client secret in a +// chunk, so chunks routinely carry dozens of candidate pairs. Implementing ResultVerifier +// lets the verification cache verify only the pairs it has not seen before instead of +// re-verifying the whole product whenever one pair is novel. +var _ detectors.ResultVerifier = (*Scanner)(nil) + var ( // Oauth2 client ID and secret oauth2ClientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-zA-Z0-9]{20})\b`) oauth2ClientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{40})\b`) + + // tokenURL is a variable rather than a direct reference to github.Endpoint.TokenURL so + // that verification tests can redirect it at an httptest server. + tokenURL = github.Endpoint.TokenURL ) const ( @@ -59,19 +69,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "rotation_guide": "https://howtorotate.com/docs/tutorials/github/", } - config := &clientcredentials.Config{ - ClientID: idMatch[1], - ClientSecret: secretMatch[1], - TokenURL: github.Endpoint.TokenURL, - } + // Verification is delegated so that this path and the verification cache's + // per-result path share one implementation. if verify { - _, err := config.Token(ctx) - // if client id and client secret is correct, it will return bad verification code error as we do not pass any verification code - // docs: https://docs.github.com/en/apps/oauth-apps/maintaining-oauth-apps/troubleshooting-oauth-app-access-token-request-errors#bad-verification-code - if err != nil && strings.Contains(err.Error(), githubBadVerificationCodeError) { - // mark result as verified only in case of bad verification code error, for any other error the result will be unverified - s1.Verified = true - } + s.VerifyResult(ctx, &s1) } results = append(results, s1) @@ -81,6 +82,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return } +// VerifyResult verifies a single client ID / client secret pair. +// the verification cache calls it directly for results that missed the cache, which +// is what keeps a chunk of many pairs from re-verifying pairs whose status is already known. +func (s Scanner) VerifyResult(ctx context.Context, result *detectors.Result) { + clientID := result.SecretParts["id"] + clientSecret := result.SecretParts["secret"] + clientCredentials := &clientcredentials.Config{ + ClientID: clientID, + ClientSecret: clientSecret, + TokenURL: tokenURL, + } + + _, err := clientCredentials.Token(ctx) + // if client id and client secret is correct, it will return bad verification code error as we do not pass any verification code + // docs: https://docs.github.com/en/apps/oauth-apps/maintaining-oauth-apps/troubleshooting-oauth-app-access-token-request-errors#bad-verification-code + if err != nil && strings.Contains(err.Error(), githubBadVerificationCodeError) { + // mark result as verified only in case of bad verification code error, for any other error the result will be unverified + result.Verified = true + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_GitHubOauth2 } diff --git a/pkg/detectors/github_oauth2/github_oauth2_test.go b/pkg/detectors/github_oauth2/github_oauth2_test.go index ae95bebad20a..daf072bed316 100644 --- a/pkg/detectors/github_oauth2/github_oauth2_test.go +++ b/pkg/detectors/github_oauth2/github_oauth2_test.go @@ -2,9 +2,13 @@ package github_oauth2 import ( "context" + "net/http" + "net/http/httptest" "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "golang.org/x/oauth2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -74,3 +78,92 @@ func TestGithubOAuth2_Pattern(t *testing.T) { }) } } + +// TestGithubOAuth2_VerifyResult covers the single-pair verification entry point that the +// verification cache calls for cache misses. Verification is pointed at a local server by +// reassigning tokenURL and by handing x/oauth2 a client through the context, since +// clientcredentials resolves its HTTP client from oauth2.HTTPClient. +func TestGithubOAuth2_VerifyResult(t *testing.T) { + const ( + clientID = "9c14koUc3f04PrzlpCcU" + clientSecret = "caa3224b5ddb83924e6a487ee3f6543bae428309" + ) + + tests := []struct { + name string + status int + contentType string + body string + wantVerified bool + }{ + { + // GitHub answers a valid ID/secret pair with bad_verification_code, because the + // client_credentials grant supplies no verification code. It reports this with + // HTTP 200 rather than a 4xx, which x/oauth2 still surfaces as a RetrieveError + // because the body carries an error field. + name: "live credential - bad_verification_code", + status: http.StatusOK, + contentType: "application/x-www-form-urlencoded", + body: "error=bad_verification_code&error_description=The+code+passed+is+incorrect+or+expired.", + wantVerified: true, + }, + { + name: "dead credential - incorrect_client_credentials", + status: http.StatusUnauthorized, + contentType: "application/x-www-form-urlencoded", + body: "error=incorrect_client_credentials", + wantVerified: false, + }, + { + // A transient server failure currently lands here as "not verified" rather than + // "unknown", because verification never calls SetVerificationError. That is a + // known false negative tracked on its own ticket; this case pins today's behavior + // so that changing it later is deliberate rather than accidental. + name: "server error - recorded unverified rather than unknown", + status: http.StatusInternalServerError, + contentType: "text/plain", + body: "boom", + wantVerified: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var gotID, gotSecret string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // clientcredentials presents the pair either as basic auth or as form values, + // depending on which auth style x/oauth2 is probing, so accept both. + if id, secret, ok := r.BasicAuth(); ok { + gotID, gotSecret = id, secret + } else if err := r.ParseForm(); err == nil { + gotID, gotSecret = r.PostFormValue("client_id"), r.PostFormValue("client_secret") + } + w.Header().Set("Content-Type", test.contentType) + w.WriteHeader(test.status) + _, _ = w.Write([]byte(test.body)) + })) + defer server.Close() + + originalTokenURL := tokenURL + tokenURL = server.URL + t.Cleanup(func() { tokenURL = originalTokenURL }) + + // x/oauth2 resolves its HTTP client from the context, which is how verification is + // redirected at the test server without the detector knowing about it. + ctx := context.WithValue(context.Background(), oauth2.HTTPClient, server.Client()) + + result := detectors.Result{ + Raw: []byte(clientID), + RawV2: []byte(clientID + clientSecret), + SecretParts: map[string]string{"id": clientID, "secret": clientSecret}, + } + Scanner{}.VerifyResult(ctx, &result) + + assert.Equal(t, test.wantVerified, result.Verified) + // Confirms the pair travelled from SecretParts into the token request, which is + // the contract the verification cache relies on when it verifies a lone result. + assert.Equal(t, clientID, gotID) + assert.Equal(t, clientSecret, gotSecret) + }) + } +} diff --git a/pkg/detectors/githubapp/githubapp.go b/pkg/detectors/githubapp/githubapp.go index 25f82b0b16af..dd3b1840311b 100644 --- a/pkg/detectors/githubapp/githubapp.go +++ b/pkg/detectors/githubapp/githubapp.go @@ -86,7 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenString)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.go index 793a44de1988..e77633eb0446 100644 --- a/pkg/detectors/gitlab/v1/gitlab.go +++ b/pkg/detectors/gitlab/v1/gitlab.go @@ -37,8 +37,16 @@ var ( keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b([a-zA-Z0-9][a-zA-Z0-9\-=_]{19,21})\b`) BlockedUserMessage = "403 Forbidden - Your account has been blocked" + + // GitLab's forbidden! helper formats every reasoned 403 as "403 Forbidden - ". + forbiddenWithReasonPrefix = "403 Forbidden - " ) +type gitlabErrorResponse struct { + Message string `json:"message"` + Error string `json:"error"` +} + func (s Scanner) getClient() *http.Client { if s.client != nil { return s.client @@ -88,6 +96,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: map[string]string{ + "key": resMatch, + "host": endpoint, + }, } if verify { @@ -95,16 +107,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified maps.Copy(s1.ExtraData, extraData) - s1.SetVerificationError(verificationErr) + s1.SetVerificationError(verificationErr, resMatch) - // for verified keys set the analysis info + // for verified keys break out of the endpoint loop to continue to next secret if s1.Verified { - s1.SecretParts = map[string]string{ - "key": resMatch, - "host": endpoint, - } - - // if secret is verified with one endpoint, break the loop to continue to next secret results = append(results, s1) break } @@ -134,7 +140,7 @@ func VerifyGitlab(ctx context.Context, client *http.Client, baseEndpoint, resMat return false, nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() bodyBytes, err := io.ReadAll(res.Body) if err != nil { @@ -142,22 +148,34 @@ func VerifyGitlab(ctx context.Context, client *http.Client, baseEndpoint, resMat } // 200 means good key and has `read_user` scope - // 403 means good key but not the right scope + // 403 from GitLab means good key, but either the scope or the account state denies access // 401 is bad key switch res.StatusCode { case http.StatusOK: return json.Valid(bodyBytes), nil, nil case http.StatusForbidden: - // check if the user account is blocked or not - stringBody := string(bodyBytes) - if strings.Contains(stringBody, BlockedUserMessage) { - return true, map[string]string{ - "blocked": "True", - }, nil + var errResp gitlabErrorResponse + if err := json.Unmarshal(bodyBytes, &errResp); err != nil { + return false, nil, fmt.Errorf("unexpected 403 response, unable to verify token") + } + + switch { + // GitLab only rejects an account (blocked, deactivated, password expired, unconfirmed + // email, etc.) after the token has resolved to a user. + case strings.HasPrefix(errResp.Message, forbiddenWithReasonPrefix): + extraData := map[string]string{ + "access_denied_reason": strings.TrimPrefix(errResp.Message, forbiddenWithReasonPrefix), + } + if strings.HasPrefix(errResp.Message, BlockedUserMessage) { + extraData["blocked"] = "True" + } + return true, extraData, nil + case errResp.Error == "insufficient_scope", errResp.Error == "insufficient_granular_scope": + // Good key but not the right scope + return true, nil, nil } - // Good key but not the right scope - return true, nil, nil + return false, nil, fmt.Errorf("unexpected 403 response, unable to verify token") case http.StatusUnauthorized: // Nothing to do; zero values are the ones we want return false, nil, nil diff --git a/pkg/detectors/gitlab/v1/gitlab_integration_test.go b/pkg/detectors/gitlab/v1/gitlab_integration_test.go index 5a4d8230734f..60a1a3e8d9c8 100644 --- a/pkg/detectors/gitlab/v1/gitlab_integration_test.go +++ b/pkg/detectors/gitlab/v1/gitlab_integration_test.go @@ -143,7 +143,7 @@ func TestGitlab_FromChunk(t *testing.T) { }, { name: "found, good key but wrong scope", - s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + s: Scanner{client: common.ConstantResponseHttpClient(403, `{"error":"insufficient_scope","error_description":"The request requires higher privileges than provided by the access token.","scope":"read_user api read_api"}`)}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), @@ -161,6 +161,27 @@ func TestGitlab_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found but ambiguous 403 (not insufficient_scope)", + s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Gitlab, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/", + "version": "1", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, { name: "not found", s: Scanner{}, diff --git a/pkg/detectors/gitlab/v1/gitlab_v1_test.go b/pkg/detectors/gitlab/v1/gitlab_v1_test.go index b5d479bf76f2..410d599b2589 100644 --- a/pkg/detectors/gitlab/v1/gitlab_v1_test.go +++ b/pkg/detectors/gitlab/v1/gitlab_v1_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -89,3 +90,81 @@ func TestGitLab_Pattern(t *testing.T) { }) } } + +func TestVerifyGitlab_Forbidden(t *testing.T) { + tests := []struct { + name string + body string + wantVerified bool + wantExtraData map[string]string + wantErr bool + }{ + { + name: "blocked account", + body: `{"message":"403 Forbidden - Your account has been blocked."}`, + wantVerified: true, + wantExtraData: map[string]string{ + "access_denied_reason": "Your account has been blocked.", + "blocked": "True", + }, + }, + { + name: "password expired", + body: `{"message":"403 Forbidden - Your password expired. Please access GitLab from a web browser to update your password."}`, + wantVerified: true, + wantExtraData: map[string]string{ + "access_denied_reason": "Your password expired. Please access GitLab from a web browser to update your password.", + }, + }, + { + name: "unconfirmed email", + body: `{"message":"403 Forbidden - Your primary email address is not confirmed. Please check your inbox for the confirmation instructions."}`, + wantVerified: true, + wantExtraData: map[string]string{ + "access_denied_reason": "Your primary email address is not confirmed. Please check your inbox for the confirmation instructions.", + }, + }, + { + name: "insufficient scope", + body: `{"error":"insufficient_scope","error_description":"The request requires higher privileges than provided by the access token.","scope":"read_user api read_api"}`, + wantVerified: true, + }, + { + name: "insufficient granular scope", + body: `{"error":"insufficient_granular_scope","error_description":"The request requires higher privileges than provided by the access token."}`, + wantVerified: true, + }, + { + name: "empty body", + body: "", + wantErr: true, + }, + { + name: "non-JSON body from a proxy or WAF", + body: "403 Forbidden", + wantErr: true, + }, + { + name: "JSON without a GitLab reason", + body: `{"message":"403 Forbidden"}`, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := common.ConstantResponseHttpClient(403, test.body) + verified, extraData, err := VerifyGitlab(context.Background(), client, "https://gitlab.com", "token") + + if (err != nil) != test.wantErr { + t.Fatalf("error = %v, wantErr %v", err, test.wantErr) + } + if verified != test.wantVerified { + t.Errorf("verified = %v, want %v", verified, test.wantVerified) + } + if diff := cmp.Diff(test.wantExtraData, extraData); diff != "" { + t.Errorf("extraData diff: (-want +got)\n%s", diff) + } + }) + } +} diff --git a/pkg/detectors/gitlab/v2/gitlab_integration_test.go b/pkg/detectors/gitlab/v2/gitlab_integration_test.go index beb684395c8d..8689e4edec0a 100644 --- a/pkg/detectors/gitlab/v2/gitlab_integration_test.go +++ b/pkg/detectors/gitlab/v2/gitlab_integration_test.go @@ -328,7 +328,7 @@ func TestGitlabV2_FromChunk(t *testing.T) { }, { name: "found, good key but wrong scope", - s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + s: Scanner{client: common.ConstantResponseHttpClient(403, `{"error":"insufficient_scope","error_description":"The request requires higher privileges than provided by the access token.","scope":"read_user api read_api"}`)}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), @@ -346,6 +346,27 @@ func TestGitlabV2_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found but ambiguous 403 (not insufficient_scope)", + s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Gitlab, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/", + "version": "2", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, { name: "not found", s: Scanner{}, diff --git a/pkg/detectors/gitlab/v2/gitlab_v2.go b/pkg/detectors/gitlab/v2/gitlab_v2.go index 8fcedad2e664..cef9fd22ff4d 100644 --- a/pkg/detectors/gitlab/v2/gitlab_v2.go +++ b/pkg/detectors/gitlab/v2/gitlab_v2.go @@ -61,6 +61,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for _, match := range matches { resMatch := strings.TrimSpace(match[1]) + if detectors.StringShannonEntropy(resMatch) < 3.6 { + continue + } + for _, endpoint := range s.Endpoints() { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Gitlab, @@ -70,6 +74,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: map[string]string{ + "key": resMatch, + "host": endpoint, + }, } if verify { @@ -78,16 +86,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified maps.Copy(s1.ExtraData, extraData) - s1.SetVerificationError(verificationErr) + s1.SetVerificationError(verificationErr, resMatch) - // for verified keys set the analysis info + // for verified keys break out of the endpoint loop to continue to next secret if s1.Verified { - s1.SecretParts = map[string]string{ - "key": resMatch, - "host": endpoint, - } - - // if secret is verified with one endpoint, break the loop to continue to next secret results = append(results, s1) break } diff --git a/pkg/detectors/gitlab/v2/gitlab_v2_test.go b/pkg/detectors/gitlab/v2/gitlab_v2_test.go index 05f8e815fa82..701525f28516 100644 --- a/pkg/detectors/gitlab/v2/gitlab_v2_test.go +++ b/pkg/detectors/gitlab/v2/gitlab_v2_test.go @@ -39,6 +39,11 @@ func TestGitLab_Pattern(t *testing.T) { }]`, want: []string{"glpat-W6fYSu70dPEo5w_SwbHWgQhttps://gitlab.com"}, }, + { + name: "low-entropy placeholder", + input: `gitlab_secret = "glpat-xxxxxxxxxxxxxxxxxxxx"`, + want: nil, + }, } for _, test := range tests { diff --git a/pkg/detectors/gitlab/v3/gitlab_v3.go b/pkg/detectors/gitlab/v3/gitlab_v3.go index 1972b6185f86..f37098dde591 100644 --- a/pkg/detectors/gitlab/v3/gitlab_v3.go +++ b/pkg/detectors/gitlab/v3/gitlab_v3.go @@ -62,6 +62,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for _, match := range matches { resMatch := strings.TrimSpace(match[1]) + if detectors.StringShannonEntropy(resMatch) < 3.6 { + continue + } + for _, endpoint := range s.Endpoints() { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Gitlab, @@ -71,6 +75,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: map[string]string{ + "key": resMatch, + "host": endpoint, + }, } if verify { @@ -79,16 +87,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified maps.Copy(s1.ExtraData, extraData) - s1.SetVerificationError(verificationErr) + s1.SetVerificationError(verificationErr, resMatch) - // for verified keys set the analysis info + // for verified keys break out of the endpoint loop to continue to next secret if s1.Verified { - s1.SecretParts = map[string]string{ - "key": resMatch, - "host": endpoint, - } - - // if secret is verified with one endpoint, break the loop to continue to next secret results = append(results, s1) break } diff --git a/pkg/detectors/gitlab/v3/gitlab_v3_integration_test.go b/pkg/detectors/gitlab/v3/gitlab_v3_integration_test.go index 3db44ae0a217..eecdd2dd77c1 100644 --- a/pkg/detectors/gitlab/v3/gitlab_v3_integration_test.go +++ b/pkg/detectors/gitlab/v3/gitlab_v3_integration_test.go @@ -124,7 +124,7 @@ func TestGitlabV3_FromChunk(t *testing.T) { }, { name: "found, good key but wrong scope", - s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + s: Scanner{client: common.ConstantResponseHttpClient(403, `{"error":"insufficient_scope","error_description":"The request requires higher privileges than provided by the access token.","scope":"read_user api read_api"}`)}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), @@ -142,6 +142,27 @@ func TestGitlabV3_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found but ambiguous 403 (not insufficient_scope)", + s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Gitlab, + Verified: false, + ExtraData: map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/", + "version": "3", + }, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, { name: "not found", s: Scanner{}, diff --git a/pkg/detectors/gitlab/v3/gitlab_v3_test.go b/pkg/detectors/gitlab/v3/gitlab_v3_test.go index 4a901baf7c9b..80f84dd20149 100644 --- a/pkg/detectors/gitlab/v3/gitlab_v3_test.go +++ b/pkg/detectors/gitlab/v3/gitlab_v3_test.go @@ -39,6 +39,11 @@ func TestGitLab_Pattern(t *testing.T) { }]`, want: []string{"glpat-3fZ1p5y4XWcCvMGVlfakeW86MQp1Oml3Ymg0Cw.01.1203afakehttps://gitlab.com"}, }, + { + name: "low-entropy placeholder", + input: `gitlab_secret = "glpat-xxxxxxxxxxxxxxxxxxxxxxxxxxx.xx.xxxxxxxxx"`, + want: nil, + }, } for _, test := range tests { diff --git a/pkg/detectors/gitlaboauth2/gitlaboauth2.go b/pkg/detectors/gitlaboauth2/gitlaboauth2.go new file mode 100644 index 000000000000..814defed73db --- /dev/null +++ b/pkg/detectors/gitlaboauth2/gitlaboauth2.go @@ -0,0 +1,166 @@ +package gitlaboauth2 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.EndpointSetter + detectors.DefaultMultiPartCredentialProvider +} + +func (Scanner) CloudEndpoint() string { return "https://gitlab.com" } + +var ( + // Ensure the Scanner satisfies the interfaces at compile time. + _ detectors.Detector = (*Scanner)(nil) + _ detectors.EndpointCustomizer = (*Scanner)(nil) + _ detectors.MultiPartCredentialProvider = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() + clientIdPat = regexp.MustCompile( + detectors.PrefixRegex([]string{"application_id", "client_id", "app_id", "id"}) + `\b([0-9a-f]{64})\b`) + clientSecretPat = regexp.MustCompile(`\b(gloas-[0-9a-f]{64})\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { + return []string{"gloas-"} +} + +// FromData will find and optionally verify GitLab OAuth secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueIdMatches := make(map[string]struct{}) + for _, match := range clientIdPat.FindAllStringSubmatch(dataStr, -1) { + uniqueIdMatches[match[1]] = struct{}{} + } + + uniqueSecretMatches := make(map[string]struct{}) + for _, match := range clientSecretPat.FindAllStringSubmatch(dataStr, -1) { + uniqueSecretMatches[match[1]] = struct{}{} + } + + for clientId := range uniqueIdMatches { + secretLoop: + for clientSecret := range uniqueSecretMatches { + for _, endpoint := range s.Endpoints() { + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(clientSecret), + RawV2: []byte(clientId + clientSecret + endpoint), + SecretParts: map[string]string{ + "client_id": clientId, + "client_secret": clientSecret, + "endpoint": endpoint, + }, + } + + if verify { + isVerified, verificationErr := verifyMatch( + ctx, s.getClient(), endpoint, clientId, clientSecret, + ) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr) + + if s1.Verified { + // A client_id is bound to a single secret; skip remaining + // endpoints and secrets for this client_id. + results = append(results, s1) + break secretLoop + } + } + + results = append(results, s1) + } + } + } + + return +} + +func verifyMatch(ctx context.Context, client *http.Client, endpoint string, clientId string, clientSecret string) (bool, error) { + url := endpoint + "/oauth/token" + payload := strings.NewReader("grant_type=client_credentials&client_id=" + clientId + + "&client_secret=" + clientSecret) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, payload) + if err != nil { + return false, err + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + // We use grant_type=client_credentials which GitLab doesn't support for OAuth apps, + // so even valid credentials never return 200. Instead, GitLab validates credentials + // before checking grant type: + // + // - 400 with "invalid_scope": credentials are valid (grant type rejected after auth passed) + // - 401 with "invalid_client": credentials are invalid + // + // Any other status (e.g. 422 for apps with `api` scope) is treated as inconclusive + // since we cannot determine credential validity, and falls through to the default case. + switch res.StatusCode { + case http.StatusBadRequest: + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + var errResp struct { + Error string `json:"error"` + } + if err := json.Unmarshal(bodyBytes, &errResp); err != nil { + return false, err + } + + if errResp.Error == "invalid_scope" { + return true, nil + } + + return false, fmt.Errorf("unexpected error in response: %s", errResp.Error) + + case http.StatusUnauthorized: + return false, nil + + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_GitLabOauth2 +} + +func (s Scanner) Description() string { + return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab OAuth application credentials can be used to access GitLab APIs on behalf of users." +} diff --git a/pkg/detectors/gitlaboauth2/gitlaboauth2_integration_test.go b/pkg/detectors/gitlaboauth2/gitlaboauth2_integration_test.go new file mode 100644 index 000000000000..df8a33719136 --- /dev/null +++ b/pkg/detectors/gitlaboauth2/gitlaboauth2_integration_test.go @@ -0,0 +1,185 @@ +//go:build detectors +// +build detectors + +package gitlaboauth2 + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestGitlabOauth_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + clientId := testSecrets.MustGetField("GITLABOAUTH_CLIENT_ID") + clientSecret := testSecrets.MustGetField("GITLABOAUTH_CLIENT_SECRET") + inactiveClientId := testSecrets.MustGetField("GITLABOAUTH_CLIENT_ID_INACTIVE") + inactiveClientSecret := testSecrets.MustGetField("GITLABOAUTH_CLIENT_SECRET_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: ctx, + data: []byte(fmt.Sprintf(` + gitlab: + client_id: %s + client_secret: %s + `, clientId, clientSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_GitLabOauth2, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: ctx, + data: []byte(fmt.Sprintf(` + gitlab: + client_id: %s + client_secret: %s + `, inactiveClientId, inactiveClientSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_GitLabOauth2, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: ctx, + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: ctx, + data: []byte(fmt.Sprintf(` + gitlab: + client_id: %s + client_secret: %s + `, clientId, clientSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_GitLabOauth2, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: ctx, + data: []byte(fmt.Sprintf(` + gitlab: + client_id: %s + client_secret: %s + `, clientId, clientSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_GitLabOauth2, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.s.SetCloudEndpoint(tt.s.CloudEndpoint()) + tt.s.UseCloudEndpoint(true) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("GitlabOauth.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, + "Raw", "RawV2", "verificationError", "ExtraData", "primarySecret", "SecretParts") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("GitlabOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + s.SetCloudEndpoint(s.CloudEndpoint()) + s.UseCloudEndpoint(true) + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/gitlaboauth2/gitlaboauth2_test.go b/pkg/detectors/gitlaboauth2/gitlaboauth2_test.go new file mode 100644 index 000000000000..5978a5586a5b --- /dev/null +++ b/pkg/detectors/gitlaboauth2/gitlaboauth2_test.go @@ -0,0 +1,138 @@ +package gitlaboauth2 + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestGitlabOauth_Pattern(t *testing.T) { + d := Scanner{} + d.SetCloudEndpoint("https://gitlab.com") + d.UseCloudEndpoint(true) + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - client_id and client_secret", + input: ` + gitlab: + client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a + client_secret: gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216 + `, + want: []string{"04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098agloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216https://gitlab.com"}, + }, + { + name: "valid pattern - application_id prefix", + input: ` + GITLAB_APPLICATION_ID=99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975df + GITLAB_SECRET=gloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321 + `, + want: []string{"99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975dfgloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321https://gitlab.com"}, + }, + { + name: "valid pattern - JSON format", + input: ` + { + "app_id": "763c4e64f4c40dd070010617639cc11e37bbaf1a798503dd96ee5e6852754862", + "secret": "gloas-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + } + `, + want: []string{"763c4e64f4c40dd070010617639cc11e37bbaf1a798503dd96ee5e6852754862gloas-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdefhttps://gitlab.com"}, + }, + { + name: "multiple matches", + input: ` + # Production + client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a + client_secret: gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216 + + # Staging + client_id: 99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975df + client_secret: gloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321 + `, + want: []string{ + "04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098agloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216https://gitlab.com", + "04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098agloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321https://gitlab.com", + "99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975dfgloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216https://gitlab.com", + "99f0b46fc9241b7fa4b7d567044fab74a5f00ac0f12244ccfed1ea67d4a975dfgloas-35fa9094e834aafb153bc17f1a31f48071af915c2ccf2f890b6714b954896321https://gitlab.com", + }, + }, + { + name: "invalid pattern - wrong secret prefix", + input: ` + client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a + client_secret: glpat-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216 + `, + want: []string{}, + }, + { + name: "invalid pattern - secret too short", + input: ` + client_id: 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a + client_secret: gloas-8406980541370e5bd4f04c5da232c2cd + `, + want: []string{}, + }, + { + name: "invalid pattern - client_id too short", + input: ` + client_id: 04fcf956cb6c5f4b106f3a4eca76eaf7 + client_secret: gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216 + `, + want: []string{}, + }, + { + name: "invalid pattern - no client_id context prefix", + input: ` + 04fcf956cb6c5f4b106f3a4eca76eaf70e8c5d07a976ecf3baff9ac778a0098a + gloas-8406980541370e5bd4f04c5da232c2cdabe7fa3959eb1757eeef7299e2458216 + `, + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(test.want) > 0 && len(matchedDetectors) == 0 { + t.Errorf("keywords were not matched: %v", d.Keywords()) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results)) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/gitter/gitter.go b/pkg/detectors/gitter/gitter.go index ca047ecea79f..c167897d5022 100644 --- a/pkg/detectors/gitter/gitter.go +++ b/pkg/detectors/gitter/gitter.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Gitter, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/glassnode/glassnode.go b/pkg/detectors/glassnode/glassnode.go index 4b08a50d43ed..0ca330818118 100644 --- a/pkg/detectors/glassnode/glassnode.go +++ b/pkg/detectors/glassnode/glassnode.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gocanvas/gocanvas.go b/pkg/detectors/gocanvas/gocanvas.go index 0fbfeb62c40b..88b9ddf4e0c4 100644 --- a/pkg/detectors/gocanvas/gocanvas.go +++ b/pkg/detectors/gocanvas/gocanvas.go @@ -59,31 +59,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := url.Values{} - payload.Add("username", emailMatch) - - req, err := http.NewRequestWithContext(ctx, "GET", "https://www.gocanvas.com/apiv2/forms.xml", strings.NewReader(payload.Encode())) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - body, errBody := io.ReadAll(res.Body) - - if errBody == nil { - response := Response{} - if err := xml.Unmarshal(body, &response); err != nil { - continue - } - - if res.StatusCode >= 200 && res.StatusCode < 300 && response.Error == nil { - s1.Verified = true - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, emailMatch, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, emailMatch, resMatch) } results = append(results, s1) @@ -93,6 +71,38 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, emailMatch string, resMatch string) (bool, error) { + payload := url.Values{} + payload.Add("username", emailMatch) + + req, err := http.NewRequestWithContext(ctx, "GET", "https://www.gocanvas.com/apiv2/forms.xml", strings.NewReader(payload.Encode())) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + body, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + response := Response{} + if err := xml.Unmarshal(body, &response); err != nil { + return false, err + } + + if res.StatusCode >= 200 && res.StatusCode < 300 && response.Error == nil { + return true, nil + } + + return false, nil +} + type Response struct { Error []struct { ErrorCode int `xml:"ErrorCode"` diff --git a/pkg/detectors/gocardless/gocardless.go b/pkg/detectors/gocardless/gocardless.go index 8bc90a8eb8cb..7edc19c19c0a 100644 --- a/pkg/detectors/gocardless/gocardless.go +++ b/pkg/detectors/gocardless/gocardless.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("GoCardless-Version", "2015-07-06") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/goodday/goodday.go b/pkg/detectors/goodday/goodday.go index 11aff376bdfc..c41a8f8d0f08 100644 --- a/pkg/detectors/goodday/goodday.go +++ b/pkg/detectors/goodday/goodday.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("gd-api-token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/googleoauth2/googleoauth2_access_token.go b/pkg/detectors/googleoauth2/googleoauth2_access_token.go index 2939e75409db..a7c1845b01af 100644 --- a/pkg/detectors/googleoauth2/googleoauth2_access_token.go +++ b/pkg/detectors/googleoauth2/googleoauth2_access_token.go @@ -88,7 +88,8 @@ func (s Scanner) verify(ctx context.Context, token string) (bool, map[string]str _, _ = io.Copy(io.Discard, res.Body) }() - if res.StatusCode == http.StatusOK { + switch res.StatusCode { + case http.StatusOK: var token tokenInfo if err := json.NewDecoder(res.Body).Decode(&token); err != nil { return false, nil, fmt.Errorf("failed to decode response: %w", err) @@ -106,7 +107,7 @@ func (s Scanner) verify(ctx context.Context, token string) (bool, map[string]str extraData["expires_at"] = time.Unix(exp, 0).String() } return true, extraData, nil - } else if res.StatusCode == http.StatusBadRequest { + case http.StatusBadRequest: var errInfo errorInfo if err := json.NewDecoder(res.Body).Decode(&errInfo); err != nil { return false, nil, fmt.Errorf("failed to decode response: %w", err) @@ -118,7 +119,7 @@ func (s Scanner) verify(ctx context.Context, token string) (bool, map[string]str } else { return false, nil, fmt.Errorf("unexpected error description '%s' for %s", errInfo.Error, req.URL) } - } else { + default: return false, nil, fmt.Errorf("unexpected response %d for %s", res.StatusCode, req.URL) } } diff --git a/pkg/detectors/grafana/grafana.go b/pkg/detectors/grafana/grafana.go index 3a43bd801c1e..00cdc40462c1 100644 --- a/pkg/detectors/grafana/grafana.go +++ b/pkg/detectors/grafana/grafana.go @@ -96,16 +96,38 @@ func verifyGrafanaKey(ctx context.Context, client *http.Client, token string) (b }() switch resp.StatusCode { - case http.StatusOK: + case http.StatusOK, http.StatusForbidden: + // 200: token is valid and has permission to list tokens. + // 403: token is valid (authenticated) but lacks permission for this + // resource. Either way the credentials are genuine, so it's verified. return true, nil case http.StatusUnauthorized: + // Grafana returns 401 for two very different cases that can only be + // told apart by the response body: + // + // 1. A valid token that authenticated successfully but lacks the + // accesspolicies:read scope. The body reports a permission/scope + // error, e.g.: + // {"code":"Unauthorized","message":"invalid permission: access + // policy missing required scope [accesspolicies:read], ..."} + // + // 2. An invalid/revoked token that failed authentication, e.g.: + // {"code":"InvalidCredentials","message":"Token could not be parsed"} + // + // A permission/scope error can only be produced after authentication + // succeeds, so use that as the positive signal. We deliberately do not + // match the looser "Unauthorized" string because revoked tokens can + // also surface it, which previously caused false positives. bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return false, err } + body := string(bodyBytes) - // token is valid but has restricted permissions - return strings.Contains(string(bodyBytes), "Unauthorized"), nil + if strings.Contains(body, "invalid permission") || strings.Contains(body, "required scope") { + return true, nil + } + return false, nil default: return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } diff --git a/pkg/detectors/grafana/grafana_test.go b/pkg/detectors/grafana/grafana_test.go index fa459a176f63..11fc608c16ad 100644 --- a/pkg/detectors/grafana/grafana_test.go +++ b/pkg/detectors/grafana/grafana_test.go @@ -2,10 +2,12 @@ package grafana import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -86,3 +88,84 @@ func TestGrafana_Pattern(t *testing.T) { }) } } + +func TestGrafana_Verification(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantVerified bool + wantErr bool + }{ + { + name: "200 OK is verified", + statusCode: 200, + body: `[{"id":"abc"}]`, + wantVerified: true, + }, + { + name: "403 Forbidden (valid token, restricted permissions) is verified", + statusCode: 403, + body: `{"message":"You'll need additional permissions to perform this action."}`, + wantVerified: true, + }, + { + // Valid token that authenticated but lacks accesspolicies:read scope. + // Grafana returns 401 with a permission/scope error -> verified. + name: "401 with permission/scope error (valid token) is verified", + statusCode: 401, + body: `{"code":"Unauthorized","message":"invalid permission: access policy missing required scope [accesspolicies:read], received [accesspolicies:write]","requestId":"fc800c18-fb65-4013-bb8b-2e047a698974"}`, + wantVerified: true, + }, + { + // Invalid/revoked token that failed authentication -> unverified. + name: "401 InvalidCredentials (revoked token) is unverified", + statusCode: 401, + body: `{"code":"InvalidCredentials","message":"Token could not be parsed","requestId":"538c7b37-2882-4727-9fae-6fc87321aa5c"}`, + wantVerified: false, + }, + { + // Regression: a revoked token whose body merely contains + // "Unauthorized" (but no permission/scope error) must NOT be verified. + name: "401 generic Unauthorized (revoked token) is unverified", + statusCode: 401, + body: `{"code":"Unauthorized","message":"Unauthorized"}`, + wantVerified: false, + }, + { + name: "401 with empty body is unverified", + statusCode: 401, + body: ``, + wantVerified: false, + }, + { + name: "unexpected status code returns verification error", + statusCode: 404, + body: ``, + wantVerified: false, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := Scanner{client: common.ConstantResponseHttpClient(test.statusCode, test.body)} + data := []byte(fmt.Sprintf("a grafana secret %s here", secret)) + + results, err := s.FromData(context.Background(), true, data) + if err != nil { + t.Fatalf("FromData() unexpected error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + if results[0].Verified != test.wantVerified { + t.Errorf("Verified = %v, want %v", results[0].Verified, test.wantVerified) + } + if gotErr := results[0].VerificationError() != nil; gotErr != test.wantErr { + t.Errorf("verification error present = %v, want %v (err = %v)", gotErr, test.wantErr, results[0].VerificationError()) + } + }) + } +} diff --git a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go index a98e7b8fc7e0..02fab0ed7507 100644 --- a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go +++ b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go @@ -68,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/graphcms/graphcms.go b/pkg/detectors/graphcms/graphcms.go index 73c843cf5543..6c1e1ab012ba 100644 --- a/pkg/detectors/graphcms/graphcms.go +++ b/pkg/detectors/graphcms/graphcms.go @@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/graphhopper/graphhopper.go b/pkg/detectors/graphhopper/graphhopper.go index 00a7022aef28..a528a1b5e7db 100644 --- a/pkg/detectors/graphhopper/graphhopper.go +++ b/pkg/detectors/graphhopper/graphhopper.go @@ -49,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/groovehq/groovehq.go b/pkg/detectors/groovehq/groovehq.go index bcb8ebb5a158..141a61754114 100644 --- a/pkg/detectors/groovehq/groovehq.go +++ b/pkg/detectors/groovehq/groovehq.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/groq/groq.go b/pkg/detectors/groq/groq.go index 044817221314..08b5694b2a17 100644 --- a/pkg/detectors/groq/groq.go +++ b/pkg/detectors/groq/groq.go @@ -45,6 +45,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/groq/", }, + SecretParts: map[string]string{"key": match}, } if verify { @@ -57,12 +58,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr, match) - - if isVerified { - s1.SecretParts = map[string]string{ - "key": match, - } - } } results = append(results, s1) diff --git a/pkg/detectors/gtmetrix/gtmetrix.go b/pkg/detectors/gtmetrix/gtmetrix.go index 6d989a72e865..b61e6ae8337b 100644 --- a/pkg/detectors/gtmetrix/gtmetrix.go +++ b/pkg/detectors/gtmetrix/gtmetrix.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/guardianapi/guardianapi.go b/pkg/detectors/guardianapi/guardianapi.go index 07ee12bd2006..d7255d820053 100644 --- a/pkg/detectors/guardianapi/guardianapi.go +++ b/pkg/detectors/guardianapi/guardianapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gumroad/gumroad.go b/pkg/detectors/gumroad/gumroad.go index 86cc11361f6f..6f77a49d2969 100644 --- a/pkg/detectors/gumroad/gumroad.go +++ b/pkg/detectors/gumroad/gumroad.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/guru/guru.go b/pkg/detectors/guru/guru.go index b0e564f91787..fd19de83c584 100644 --- a/pkg/detectors/guru/guru.go +++ b/pkg/detectors/guru/guru.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", encoded)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/gyazo/gyazo.go b/pkg/detectors/gyazo/gyazo.go index 02cb9825d621..6948b6a66072 100644 --- a/pkg/detectors/gyazo/gyazo.go +++ b/pkg/detectors/gyazo/gyazo.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/happyscribe/happyscribe.go b/pkg/detectors/happyscribe/happyscribe.go index a86cefe9f1e4..e210c5c79d2c 100644 --- a/pkg/detectors/happyscribe/happyscribe.go +++ b/pkg/detectors/happyscribe/happyscribe.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/harness/harness.go b/pkg/detectors/harness/harness.go index 5ff2416880de..4a847068dfeb 100644 --- a/pkg/detectors/harness/harness.go +++ b/pkg/detectors/harness/harness.go @@ -32,7 +32,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"harness"}) + `\b(pat\.[A-Za-z0-9]{22}\.[0-9a-f]{24}\.[A-Za-z0-9]{20})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"harness"}) + `\b(pat\.[A-Za-z0-9_]{22}\.[0-9a-f]{24}\.[A-Za-z0-9]{20})\b`) ) func (s Scanner) getClient() *http.Client { @@ -63,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Harness, Raw: []byte(match), + SecretParts: map[string]string{"key": match}, } if verify { @@ -72,13 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr, match) - - if isVerified { - s1.SecretParts = map[string]string{ - "key": match, - } - } - } results = append(results, s1) diff --git a/pkg/detectors/harness/harness_test.go b/pkg/detectors/harness/harness_test.go index ab6599e1185d..bfec67695985 100644 --- a/pkg/detectors/harness/harness_test.go +++ b/pkg/detectors/harness/harness_test.go @@ -12,6 +12,7 @@ import ( var ( validKey = "pat.4oXWHvYFRNOGLVpFTZGGTA.68077fc826afe36865614d58.2fFEmr57WO3zPmev3jze" + validKeyWithUnderscore = "pat.YDfcEm2LT_OUZrFZv1WVlg.6a4f91eb79dfb04b036caf48.FrDL8MGzpMygCMzZv1Kq" validKeyWithoutKeyword = `API Key Token: pat.4oXWHvYFRNOGLVpFTZGGTA.68077fc826afe36865614d58.2fFEmr57WO3zPmev3jze url |https://api.harness.io/` invalidKey = "pat.4oXWHvYFRNOGLVpFTZGGTA.6807c5bed9599c324f6368ce.usCT2fzvADwSoXzXc" @@ -31,6 +32,11 @@ func TestHarness_Pattern(t *testing.T) { input: fmt.Sprintf("%s token = '%s'", keyword, validKey), want: []string{validKey}, }, + { + name: "valid pattern with underscore in account segment", + input: fmt.Sprintf("%s = '%s'", keyword, validKeyWithUnderscore), + want: []string{validKeyWithUnderscore}, + }, { name: "valid pattern - no keyword", input: fmt.Sprintf("token = '%s'", validKeyWithoutKeyword), diff --git a/pkg/detectors/harvest/harvest.go b/pkg/detectors/harvest/harvest.go index 7f4aabc1e64f..7dc03bc7b7db 100644 --- a/pkg/detectors/harvest/harvest.go +++ b/pkg/detectors/harvest/harvest.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/hashicorpvault/common.go b/pkg/detectors/hashicorpvault/common.go new file mode 100644 index 000000000000..862f6f0fab64 --- /dev/null +++ b/pkg/detectors/hashicorpvault/common.go @@ -0,0 +1,83 @@ +// Package hashicorpvault contains logic shared by the HashiCorp Vault +// detectors (hashicorpvaulttoken, hashicorpvaultbatchtoken) for verifying +// tokens against the /v1/auth/token/lookup-self endpoint. +package hashicorpvault + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type LookupResponse struct { + Data struct { + DisplayName string `json:"display_name"` + EntityId string `json:"entity_id"` + ExpireTime string `json:"expire_time"` + Orphan bool `json:"orphan"` + Policies []string `json:"policies"` + Renewable bool `json:"renewable"` + Type string `json:"type"` + } +} + +// VerifyVaultToken looks up token against baseUrl's /v1/auth/token/lookup-self +// endpoint. detType is used to scope request deduplication when client was +// created with detectors.NewClientWithDedup; it has no effect otherwise. +func VerifyVaultToken( + ctx context.Context, + client *http.Client, + detType detector_typepb.DetectorType, + baseUrl string, + token string, +) (bool, *LookupResponse, error) { + lookupUrl, err := url.JoinPath(baseUrl, "/v1/auth/token/lookup-self") + if err != nil { + return false, nil, err + } + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + lookupUrl, + http.NoBody, + ) + if err != nil { + return false, nil, err + } + + req.Header.Set("X-Vault-Token", token) + + res, err := detectors.DoWithDedup(client, detType, fmt.Sprintf("%s:%s", baseUrl, token), req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + var resp LookupResponse + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + return false, nil, err + } + + return true, &resp, nil + + case http.StatusForbidden, http.StatusUnauthorized: + return false, nil, nil + + default: + return false, nil, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} diff --git a/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken.go b/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken.go new file mode 100644 index 000000000000..6e4abac86515 --- /dev/null +++ b/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken.go @@ -0,0 +1,121 @@ +package hashicorpbatchtoken + +import ( + "context" + "fmt" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hashicorpvault" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter +} + +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.EndpointCustomizer = (*Scanner)(nil) + +var ( + defaultClient = detectors.NewClientWithDedup(detectors.DetectorHttpClientWithNoLocalAddresses) + + // Batch tokens: hvb.<50-300 chars> + batchTokenPat = regexp.MustCompile( + `\b(hvb\.[A-Za-z0-9_.-]{50,300})(?:[^A-Za-z0-9_.-]|\z)`, + ) + + vaultUrlPat = regexp.MustCompile(`(https?:\/\/[^\s\/]*\.hashicorp\.cloud(?::\d+)?)(?:\/[^\s]*)?`) +) + +func (s Scanner) Keywords() []string { + return []string{"hvb."} +} + +func (Scanner) CloudEndpoint() string { return "" } + +func (s Scanner) Description() string { + return "This detector detects and verifies HashiCorp Vault batch tokens" +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range batchTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + var uniqueVaultUrls = make(map[string]struct{}) + for _, match := range vaultUrlPat.FindAllStringSubmatch(dataStr, -1) { + url := strings.TrimSpace(match[1]) + uniqueVaultUrls[url] = struct{}{} + } + + endpoints := make([]string, 0, len(uniqueVaultUrls)) + for endpoint := range uniqueVaultUrls { + endpoints = append(endpoints, endpoint) + } + + for _, endpoint := range s.Endpoints(endpoints...) { + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_HashiCorpVaultBatchToken, + Raw: []byte(token), + RawV2: []byte(token + ":" + endpoint), + Redacted: token[:8] + "...", + SecretParts: map[string]string{ + "key": token, + "url": endpoint, + }, + } + + if verify { + verified, verificationResp, verificationErr := hashicorpvault.VerifyVaultToken( + ctx, + s.getClient(), + s.Type(), + endpoint, + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + + if verificationResp != nil { + result.ExtraData = map[string]string{ + "orphan": fmt.Sprintf("%v", verificationResp.Data.Orphan), + "renewable": fmt.Sprintf("%v", verificationResp.Data.Renewable), + "type": verificationResp.Data.Type, + "entity_id": verificationResp.Data.EntityId, + } + result.ExtraData["policies"] = strings.Join(verificationResp.Data.Policies, ", ") + } + } + + results = append(results, result) + } + } + + return +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_HashiCorpVaultBatchToken +} diff --git a/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken_integration_test.go b/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken_integration_test.go new file mode 100644 index 000000000000..04b3308feb5a --- /dev/null +++ b/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken_integration_test.go @@ -0,0 +1,123 @@ +//go:build detectors +// +build detectors + +package hashicorpbatchtoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" +) + +func TestBatchToken_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Fetch test secrets + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets: %s", err) + } + + vaultURL := testSecrets.MustGetField("HASHICORPVAULT_CLOUD_URL") + + // This token has maximum TTL of 32days (768h), so it should still be valid by the time this test runs + // but if the test fails due to an invalid token, this is the most likely culprit and the token may need to be regenerated. + // To regenerate the token run this command in vault web cli: + // write auth/token/create type=batch policies="test-policy" ttl="768h" no_parent=true + batchToken := testSecrets.MustGetField("HASHICORPVAULT_BATCH_TOKEN") + + fakeToken := "hvb.fakeinvalidtokenaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + tests := []struct { + name string + input string + verify bool + wantTokens []string + wantVerified bool + wantVerificationErr bool + }{ + { + name: "valid batch token with URL, verify", + input: fmt.Sprintf("%s\n%s", batchToken, vaultURL), + verify: true, + wantTokens: []string{ + batchToken + ":" + vaultURL, + }, + wantVerified: true, + wantVerificationErr: false, + }, + { + name: "invalid batch token with URL, verify", + input: fmt.Sprintf("%s\n%s", fakeToken, vaultURL), + verify: true, + wantTokens: []string{ + fakeToken + ":" + vaultURL, + }, + wantVerified: false, + wantVerificationErr: false, + }, + { + name: "valid batch token with URL, no verify", + input: fmt.Sprintf("%s\n%s", batchToken, vaultURL), + verify: false, + wantTokens: []string{ + batchToken + ":" + vaultURL, + }, + wantVerified: false, + wantVerificationErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scanner := Scanner{} + scanner.UseFoundEndpoints(true) + scanner.UseCloudEndpoint(true) + + results, err := scanner.FromData(ctx, tt.verify, []byte(tt.input)) + require.NoError(t, err) + + if len(results) != len(tt.wantTokens) { + t.Fatalf("expected %d results, got %d", len(tt.wantTokens), len(results)) + } + + for i, r := range results { + if string(r.RawV2) != tt.wantTokens[i] && string(r.Raw) != tt.wantTokens[i] { + t.Errorf("expected token %s, got %s", tt.wantTokens[i], string(r.Raw)) + } + + if r.Verified != tt.wantVerified { + t.Errorf("expected verified=%v, got %v", tt.wantVerified, r.Verified) + } + + if (r.VerificationError() != nil) != tt.wantVerificationErr { + t.Errorf("expected verification error=%v, got %v", tt.wantVerificationErr, r.VerificationError()) + } + } + }) + } +} + +func BenchmarkFromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken_test.go b/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken_test.go new file mode 100644 index 000000000000..e2e36de862e8 --- /dev/null +++ b/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken/hashicorpvaultbatchtoken_test.go @@ -0,0 +1,144 @@ +package hashicorpbatchtoken + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestBatchToken_PatternWithURL(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid hvb token with vault url", + input: ` + func setupVaultClient() (*http.Client, error) { + vaultAddr := "https://vault-cluster-abc123.hashicorp.cloud:8200" + token := "hvb.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + // Look up the token to confirm it's still valid + req, err := http.NewRequest("GET", vaultAddr+"/v1/auth/token/lookup-self", http.NoBody) + if err != nil { + return nil, err + } + req.Header.Set("X-Vault-Token", token) + + client := &http.Client{} + resp, _ := client.Do(req) + defer func() { _ = resp.Body.Close() }() + + return client, nil + } + `, + want: []string{ + "hvb.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:https://vault-cluster-abc123.hashicorp.cloud:8200", + }, + }, + { + name: "valid hvb token with longer length", + input: ` + # vault-config.env + VAULT_ADDR=https://vault-cluster-xyz.hashicorp.cloud + VAULT_TOKEN=hvb.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + VAULT_NAMESPACE=admin + `, + want: []string{ + "hvb.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:https://vault-cluster-xyz.hashicorp.cloud", + }, + }, + { + name: "token only, no URL", + input: ` + func rotateVaultToken() { + // Deprecated batch token retained for the audit trail + oldToken := "hvb.cccccccccccccccccccccccccccccccccccccccccccccccccc" + log.Println("token retired:", oldToken) + } + `, + want: nil, + }, + { + name: "URL only, no token", + input: ` + func vaultHealthCheck() error { + resp, err := http.Get("https://vault-cluster-abc123.hashicorp.cloud:8200/v1/sys/health") + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + return nil + } + `, + want: nil, + }, + { + name: "invalid token - too short", + input: ` + func setupVaultClient() { + vaultAddr := "https://vault-cluster-abc123.hashicorp.cloud" + // Truncated token accidentally committed + token := "hvb.shorttoken" + client.SetToken(vaultAddr, token) + } + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 && len(test.want) > 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + d.UseFoundEndpoints(true) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken.go b/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken.go new file mode 100644 index 000000000000..1b462125c511 --- /dev/null +++ b/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken.go @@ -0,0 +1,122 @@ +package hashicorpvaulttoken + +import ( + "context" + "fmt" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hashicorpvault" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter +} + +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.EndpointCustomizer = (*Scanner)(nil) + +var ( + defaultClient = detectors.NewClientWithDedup(detectors.DetectorHttpClientWithNoLocalAddresses) + + // Vault tokens: + // newer vault tokens are around 90-120 chars (exluding the prefix "hvs.") and start with hvs. (HashiCorp Vault Service token) + // legacy tokens are around 18-40 chars (exluding the prefix "s.") and start with s. + vaultTokenPat = regexp.MustCompile( + `\b(hvs\.[A-Za-z0-9_-]{90,120}|s\.[A-Za-z0-9_-]{18,40})(?:$|[^A-Za-z0-9_-])`, + ) + + vaultUrlPat = regexp.MustCompile(`(https?:\/\/[^\s\/]*\.hashicorp\.cloud(?::\d+)?)(?:\/[^\s]*)?`) +) + +func (s Scanner) Keywords() []string { + // We cant use s. as a keyword because it is too broad so that's why we are using "vault". + return []string{"hvs.", "vault"} +} + +func (Scanner) CloudEndpoint() string { return "" } + +func (s Scanner) Description() string { + return "HashiCorp Vault is a secrets management service. Vault tokens (periodic, service, and admin) can be used to access and manage stored secrets and resources." +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range vaultTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + var uniqueVaultUrls = make(map[string]struct{}) + for _, match := range vaultUrlPat.FindAllStringSubmatch(dataStr, -1) { + url := strings.TrimSpace(match[1]) + uniqueVaultUrls[url] = struct{}{} + } + endpoints := make([]string, 0, len(uniqueVaultUrls)) + for endpoint := range uniqueVaultUrls { + endpoints = append(endpoints, endpoint) + } + + for token := range uniqueTokens { + for _, endpoint := range s.Endpoints(endpoints...) { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_HashiCorpVaultToken, + Raw: []byte(token), + RawV2: []byte(token + ":" + endpoint), + Redacted: token[:8] + "...", + SecretParts: map[string]string{ + "key": token, + "url": endpoint, + }, + } + + if verify { + verified, verificationResp, verificationErr := hashicorpvault.VerifyVaultToken( + ctx, + s.getClient(), + s.Type(), + endpoint, + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + if verificationResp != nil { + result.ExtraData = map[string]string{ + "policies": strings.Join(verificationResp.Data.Policies, ", "), + "orphan": fmt.Sprintf("%v", verificationResp.Data.Orphan), + "renewable": fmt.Sprintf("%v", verificationResp.Data.Renewable), + "type": verificationResp.Data.Type, + "entity_id": verificationResp.Data.EntityId, // can be helpful in revoking the token + } + } + } + + results = append(results, result) + } + } + + return +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_HashiCorpVaultToken +} diff --git a/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken_integration_test.go b/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken_integration_test.go new file mode 100644 index 000000000000..ada4bc3ca346 --- /dev/null +++ b/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken_integration_test.go @@ -0,0 +1,129 @@ +//go:build detectors +// +build detectors + +package hashicorpvaulttoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" +) + +func TestVaultToken_FromData_Integration(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Fetch test secrets from TruffleHog test secret storage + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets: %s", err) + } + + vaultURL := testSecrets.MustGetField("HASHICORPVAULT_CLOUD_URL") + // Token has maximum TTL of 32days (768h), so it should still be valid by the time this test runs + // but if the test fails due to an invalid token, this is the most likely culprit and the token may need to be regenerated. + // To regenerate the token run this command in vault web cli: + // write auth/token/create policies="test-policy" ttl="768h" display_name="integration-test-token" + token := testSecrets.MustGetField("HASHICORPVAULT_TOKEN") + + fakeToken := "hvs.CAESIDdRIXSyTRAmJ2nvTohWQOPZW4gtKBeuMCQ1amIgUcWtGigKImh2cy5wdEprMjdtZWNqRXJUeElXT0lXZ0lRZVQuV2JZVlgQ3_4Q" // invalid/unused token + + tests := []struct { + name string + input string + verify bool + wantTokens []string + wantVerified []bool + wantVerificationErr bool + }{ + { + name: "valid token with URL, verify", + input: fmt.Sprintf("%s\n%s", token, vaultURL), + verify: true, + wantTokens: []string{ + token + ":" + vaultURL, + }, + wantVerified: []bool{true}, + wantVerificationErr: false, + }, + { + name: "valid token without URL, verify", + input: fmt.Sprintf("%s", token), + verify: true, + wantTokens: []string{}, + wantVerified: []bool{false}, + wantVerificationErr: false, + }, + { + name: "invalid token with URL, verify", + input: fmt.Sprintf("%s\n%s", fakeToken, vaultURL), + verify: true, + wantTokens: []string{ + fakeToken + ":" + vaultURL, + }, + wantVerified: []bool{false}, + wantVerificationErr: false, // invalid tokens are not errors, just not verified + }, + { + name: "valid token with URL, no verify", + input: fmt.Sprintf("%s\n%s", token, vaultURL), + verify: false, + wantTokens: []string{ + token + ":" + vaultURL, + }, + wantVerified: []bool{false}, + wantVerificationErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scanner := Scanner{} + scanner.UseFoundEndpoints(true) + scanner.UseCloudEndpoint(true) + results, err := scanner.FromData(ctx, tt.verify, []byte(tt.input)) + require.NoError(t, err) + + if len(results) != len(tt.wantTokens) { + t.Fatalf("expected %d results, got %d", len(tt.wantTokens), len(results)) + } + + for i, r := range results { + if string(r.RawV2) != tt.wantTokens[i] && string(r.Raw) != tt.wantTokens[i] { + t.Errorf("expected token %s, got %s", tt.wantTokens[i], string(r.Raw)) + } + + if r.Verified != tt.wantVerified[i] { + t.Errorf("expected verified=%v, got %v", tt.wantVerified[i], r.Verified) + } + + if (r.VerificationError() != nil) != tt.wantVerificationErr { + t.Errorf("expected verification error=%v, got %v", tt.wantVerificationErr, r.VerificationError()) + } + } + }) + } +} + +func BenchmarkHashicorpVaultToken_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken_test.go b/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken_test.go new file mode 100644 index 000000000000..544fe1299e8e --- /dev/null +++ b/pkg/detectors/hashicorpvault/hashicorpvaulttoken/hashicorpvaulttoken_test.go @@ -0,0 +1,102 @@ +package hashicorpvaulttoken + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestVaultToken_PatternWithURL(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid hvs token with vault url", + input: ` + X-Vault-Token=hvs.CAESIL37MuN9DnoD8RIri7rEi1TnNI2DCvyXNAC5X3a6qmpxGh4KHGh2cy5DR0loRzZXWm5MQlFYNmRQcFNsb1ZxYnc + https://vault-cluster-abc123.hashicorp.cloud:8200 + `, + want: []string{ + "hvs.CAESIL37MuN9DnoD8RIri7rEi1TnNI2DCvyXNAC5X3a6qmpxGh4KHGh2cy5DR0loRzZXWm5MQlFYNmRQcFNsb1ZxYnc:https://vault-cluster-abc123.hashicorp.cloud:8200", + }, + }, + { + name: "valid legacy s token with vault url", + input: ` + s.1234567890abcdefddd + https://vault-cluster-abc123.hashicorp.cloud:8200 + `, + want: []string{ + "s.1234567890abcdefddd:https://vault-cluster-abc123.hashicorp.cloud:8200", + }, + }, + { + name: "token only, no URL", + input: ` + hvs.abcdefghijklmnopqr + `, + want: nil, + }, + { + name: "URL only, no token", + input: ` + https://vault-cluster-abc123.hashicorp.cloud:8200 + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 && len(test.want) > 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + d.UseFoundEndpoints(true) + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/hellosign/hellosign.go b/pkg/detectors/hellosign/hellosign.go index 668ba59ad012..92b15dbd7433 100644 --- a/pkg/detectors/hellosign/hellosign.go +++ b/pkg/detectors/hellosign/hellosign.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/helpcrunch/helpcrunch.go b/pkg/detectors/helpcrunch/helpcrunch.go index 4c2a0ae9f02e..cc4b18af99ac 100644 --- a/pkg/detectors/helpcrunch/helpcrunch.go +++ b/pkg/detectors/helpcrunch/helpcrunch.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/helpscout/helpscout.go b/pkg/detectors/helpscout/helpscout.go index b258c69fa9a5..a9886a22c3ca 100644 --- a/pkg/detectors/helpscout/helpscout.go +++ b/pkg/detectors/helpscout/helpscout.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "X") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/hereapi/hereapi.go b/pkg/detectors/hereapi/hereapi.go index 5e2decbbb881..e0e2467c395f 100644 --- a/pkg/detectors/hereapi/hereapi.go +++ b/pkg/detectors/hereapi/hereapi.go @@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/hive/hive.go b/pkg/detectors/hive/hive.go index fd959d5d7303..b1bfdd10d668 100644 --- a/pkg/detectors/hive/hive.go +++ b/pkg/detectors/hive/hive.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/hiveage/hiveage.go b/pkg/detectors/hiveage/hiveage.go index 00db0ad9be30..390db1e61569 100644 --- a/pkg/detectors/hiveage/hiveage.go +++ b/pkg/detectors/hiveage/hiveage.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", encoded)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/holidayapi/holidayapi.go b/pkg/detectors/holidayapi/holidayapi.go index aa1405f25478..7bb34952f809 100644 --- a/pkg/detectors/holidayapi/holidayapi.go +++ b/pkg/detectors/holidayapi/holidayapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_HolidayAPI, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/holistic/holistic.go b/pkg/detectors/holistic/holistic.go index 713e01362f6f..56cd2f0289e2 100644 --- a/pkg/detectors/holistic/holistic.go +++ b/pkg/detectors/holistic/holistic.go @@ -47,26 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.holistic.dev/api/v1/project", nil) - if err != nil { - continue - } - req.Header.Add("x-api-key", resMatch) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - - bodyString := string(bodyBytes) - errorResponse := strings.Contains(bodyString, `"data":[]`) - - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 && !errorResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -75,6 +58,35 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.holistic.dev/api/v1/project", nil) + if err != nil { + return false, err + } + req.Header.Add("x-api-key", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + bodyString := string(bodyBytes) + errorResponse := strings.Contains(bodyString, `"data":[]`) + + if res.StatusCode >= 200 && res.StatusCode < 300 && !errorResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Holistic } diff --git a/pkg/detectors/honeycomb/honeycomb.go b/pkg/detectors/honeycomb/honeycomb.go index d9b8fe7c979b..83145288e986 100644 --- a/pkg/detectors/honeycomb/honeycomb.go +++ b/pkg/detectors/honeycomb/honeycomb.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Honeycomb-Team", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/host/host.go b/pkg/detectors/host/host.go index b95ef6713d88..4cc96d295140 100644 --- a/pkg/detectors/host/host.go +++ b/pkg/detectors/host/host.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/html2pdf/html2pdf.go b/pkg/detectors/html2pdf/html2pdf.go index 2d15020febec..d60f9a030a86 100644 --- a/pkg/detectors/html2pdf/html2pdf.go +++ b/pkg/detectors/html2pdf/html2pdf.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Content-Type", "application/json") res, err := common.SaneHttpClient().Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go index e89cb0922615..141c55e72d5a 100644 --- a/pkg/detectors/http.go +++ b/pkg/detectors/http.go @@ -3,12 +3,10 @@ package detectors import ( "bytes" "context" - "errors" "fmt" "io" "net" "net/http" - "slices" "sync" "time" @@ -16,6 +14,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/feature" + "github.com/trufflesecurity/trufflehog/v3/pkg/ssrf" ) var DetectorHttpClientWithNoLocalAddresses *http.Client @@ -99,16 +98,26 @@ func NewDetectorTransport(T http.RoundTripper) http.RoundTripper { return &detectorTransport{T: T} } -func isLocalIP(ip net.IP) bool { - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { - return true - } - - return false -} - -var ErrNoLocalIP = errors.New("dialing local IP addresses is not allowed") - +// ErrNoLocalIP is returned when a client configured with WithNoLocalIP +// refuses to dial a non-public address. +// +// Deprecated: it is an alias of ssrf.ErrEgressBlocked; new code should use +// that sentinel directly. +var ErrNoLocalIP = ssrf.ErrEgressBlocked + +// WithNoLocalIP configures the client to refuse connections to non-public +// addresses: loopback, link-local (incl. cloud metadata), private, CGNAT, +// multicast, and the other special-use ranges the ssrf package classifies. +// The check runs in the guarded dialer on each resolved address, so it is +// DNS-rebinding safe (the previous implementation vetted a LookupIP result +// and then dialed the hostname again, which a rebinding resolver could race) +// and it re-runs on every redirect hop's fresh dial. +// +// Two behavior notes against the previous implementation: any DialContext +// already set on the transport is REPLACED, not chained, so custom dialers +// (SOCKS, custom resolvers) are discarded; and blocking is per resolved +// address, so a hostname resolving to both a public and a non-public record +// connects via the public one where it previously refused the whole host. func WithNoLocalIP() ClientOption { return func(c *http.Client) { if c.Transport == nil { @@ -129,28 +138,10 @@ func WithNoLocalIP() ClientOption { } } - // If the original DialContext is nil, set it to the default dialer - if transport.DialContext == nil { - transport.DialContext = defaultDialer.DialContext - } - originalDialContext := transport.DialContext - transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - host, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - - ips, err := net.LookupIP(host) - if err != nil { - return nil, err - } - - if slices.ContainsFunc(ips, isLocalIP) { - return nil, ErrNoLocalIP - } - - return originalDialContext(ctx, network, net.JoinHostPort(host, port)) - } + // The guarded dialer replaces any existing DialContext; within this + // package that is only ever nil or defaultDialer, whose settings the + // guard preserves. + transport.DialContext = ssrf.GuardDialer(defaultDialer).DialContext } } @@ -227,7 +218,7 @@ func (t *singleflightTransport) RoundTrip(req *http.Request) (*http.Response, er if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { diff --git a/pkg/detectors/http_test.go b/pkg/detectors/http_test.go index 24215196341e..ecd2e5a5daf0 100644 --- a/pkg/detectors/http_test.go +++ b/pkg/detectors/http_test.go @@ -3,7 +3,6 @@ package detectors import ( "context" "fmt" - "net" "net/http" "net/http/httptest" "sync" @@ -63,7 +62,7 @@ func TestWithNoLocalIP(t *testing.T) { conn, err := transport.DialContext(context.Background(), "tcp", "google.com:80") assert.NoError(t, err) assert.NotNil(t, conn) - conn.Close() + _ = conn.Close() }) t.Run("Allows dialing non-local IP", func(t *testing.T) { @@ -76,7 +75,7 @@ func TestWithNoLocalIP(t *testing.T) { conn, err := transport.DialContext(context.Background(), "tcp", "1.1.1.1:80") assert.NoError(t, err) assert.NotNil(t, conn) - conn.Close() + _ = conn.Close() }) t.Run("Handles invalid address", func(t *testing.T) { @@ -114,7 +113,7 @@ func TestDoWithDedup_Singleflight(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { n := atomic.AddInt32(&requestCount, 1) time.Sleep(20 * time.Millisecond) - fmt.Fprintf(w, `{"request":%d}`, n) + _, _ = fmt.Fprintf(w, `{"request":%d}`, n) })) defer server.Close() @@ -140,7 +139,7 @@ func TestDoWithDedup_Singleflight(t *testing.T) { errs[i] = err return } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() var buf [512]byte n, _ := resp.Body.Read(buf[:]) bodies[i] = string(buf[:n]) @@ -201,7 +200,7 @@ func TestDoWithDedup_WaiterContextCancelled(t *testing.T) { results[i] = result{err: err} return } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() results[i] = result{status: resp.StatusCode} }(i, ctx) } @@ -249,7 +248,7 @@ func TestDoWithDedup_FirstCallerContextCancelled(t *testing.T) { firstErr = err return } - resp.Body.Close() + _ = resp.Body.Close() }() // Cancel the first caller once the server is processing, then immediately @@ -268,7 +267,7 @@ func TestDoWithDedup_FirstCallerContextCancelled(t *testing.T) { secondErr = err return } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() secondStatus = resp.StatusCode }() @@ -300,7 +299,7 @@ func TestDoWithDedup_DeadlinePreserved(t *testing.T) { start := time.Now() resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req) if err == nil { - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } elapsed := time.Since(start) @@ -308,27 +307,3 @@ func TestDoWithDedup_DeadlinePreserved(t *testing.T) { assert.Error(t, err, "request to hanging server should fail") assert.Less(t, elapsed, time.Second, "timeout should be enforced by client deadline, not run indefinitely") } - -func TestIsLocalIP(t *testing.T) { - testCases := []struct { - name string - ip net.IP - expected bool - }{ - {"Loopback IPv4", net.ParseIP("127.0.0.1"), true}, - {"Loopback IPv6", net.ParseIP("::1"), true}, - {"Private IPv4", net.ParseIP("192.168.1.1"), true}, - {"Private IPv6", net.ParseIP("fd00::1"), true}, - {"Unspecified IPv4", net.ParseIP("0.0.0.0"), true}, - {"Unspecified IPv6", net.ParseIP("::"), true}, - {"Public IPv4", net.ParseIP("8.8.8.8"), false}, - {"Public IPv6", net.ParseIP("2001:4860:4860::8888"), false}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := isLocalIP(tc.ip) - assert.Equal(t, tc.expected, result) - }) - } -} diff --git a/pkg/detectors/huggingface/huggingface.go b/pkg/detectors/huggingface/huggingface.go index 9690d9f7718b..c09c1643268d 100644 --- a/pkg/detectors/huggingface/huggingface.go +++ b/pkg/detectors/huggingface/huggingface.go @@ -45,6 +45,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_HuggingFace, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -52,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = extraData s1.SetVerificationError(verificationErr, resMatch) - s1.SecretParts = map[string]string{"key": resMatch} } results = append(results, s1) @@ -78,7 +78,7 @@ func (s Scanner) verifyResult(ctx context.Context, apiKey string) (bool, map[str return false, nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { whoamiRes := whoamiResponse{} err := json.NewDecoder(res.Body).Decode(&whoamiRes) diff --git a/pkg/detectors/humanity/humanity.go b/pkg/detectors/humanity/humanity.go index f6247843af0a..62398de81334 100644 --- a/pkg/detectors/humanity/humanity.go +++ b/pkg/detectors/humanity/humanity.go @@ -47,25 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://www.humanity.com/api/v2/me?access_token="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - validResponse := strings.Contains(body, "name") - - if validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -74,6 +58,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://www.humanity.com/api/v2/me?access_token="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + validResponse := strings.Contains(body, "name") + + if validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Humanity } diff --git a/pkg/detectors/humioapitoken/humioapitoken.go b/pkg/detectors/humioapitoken/humioapitoken.go new file mode 100644 index 000000000000..9364b40e24aa --- /dev/null +++ b/pkg/detectors/humioapitoken/humioapitoken.go @@ -0,0 +1,168 @@ +package humioapitoken + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.EndpointSetter +} + +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.EndpointCustomizer = (*Scanner)(nil) +var _ detectors.CloudProvider = (*Scanner)(nil) + +func (Scanner) CloudEndpoint() string { return "https://cloud.us.humio.com" } + +// additionalCloudEndpoints lists Humio/LogScale SaaS regions beyond the +// primary one returned by CloudEndpoint(). The CloudProvider interface +// only supports a single endpoint, so these are passed as found +// endpoints during verification to ensure tokens from any region are +// verified even when the scanned data contains no URL. +var additionalCloudEndpoints = []string{ + "https://cloud.humio.com", +} + +var ( + defaultClient = common.SaneHttpClient() + + // API tokens are base62 strings with a tilde (~) separator: a 24- or + // 32-character prefix and a fixed 44-character suffix. The tilde + + // exact segment lengths make this format distinctive enough that no + // PrefixRegex or surrounding-context anchor is needed. + // Example: pHUw1oLASALFmt2ppNvwCR0Meo2nHQ15~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR + keyPat = regexp.MustCompile(`\b([A-Za-z0-9]{24}(?:[A-Za-z0-9]{8})?~[A-Za-z0-9]{44})\b`) +) + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_HumioAPIToken +} + +func (s Scanner) Keywords() []string { + // The tilde separator in the token format is distinctive but too common + // in source code (URLs, paths, shell, bitwise ops) to use as a keyword. + // Te regex handles precision within matched chunks. + return []string{"humio", "logscale"} +} + +func (s Scanner) Description() string { + return "Humio/CrowdStrike Falcon LogScale is a log management platform. API tokens grant read/write access to a specific LogScale repository via the REST and GraphQL APIs." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + tokenMatches := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + t := strings.TrimSpace(match[1]) + tokenMatches[t] = struct{}{} + } + + for token := range tokenMatches { + r := detectors.Result{ + DetectorType: detector_typepb.DetectorType_HumioAPIToken, + Raw: []byte(token), + SecretParts: map[string]string{"key": token}, + ExtraData: map[string]string{"token_type": humioTokenType(token)}, + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + + // Verification only targets known SaaS regions (US via + // CloudEndpoint, EU via additionalCloudEndpoints) plus any + // endpoints explicitly configured by the operator (e.g. + // self-hosted instances). URLs discovered in scanned data are + // not used — arbitrary hosts can return 403 for non-auth + // reasons (WAF, CDN), causing false verification. + // + // Only non-nil errors overwrite lastErr so that a clean 401 + // from the wrong region doesn't erase a transient failure + // (500, timeout) from a region that might be authoritative. + var lastErr error + for _, baseURL := range s.Endpoints(additionalCloudEndpoints...) { + isVerified, vErr := verifyAPIToken(ctx, client, baseURL, token) + r.Verified = isVerified + if vErr != nil { + lastErr = vErr + } + if isVerified { + lastErr = nil + r.ExtraData["endpoint"] = baseURL + break + } + } + r.SetVerificationError(lastErr, token) + } + + results = append(results, r) + } + + return results, nil +} + +// humioTokenType classifies the token based on prefix length: 24-char +// prefixes are personal API tokens (PATs), 32-char prefixes are +// repository/view API tokens. +func humioTokenType(token string) string { + idx := strings.Index(token, "~") + if idx == 24 { + return "Personal API Token" + } + return "Repository API Token" +} + +// verifyAPIToken hits the health-json endpoint, which is read-only and +// produces no side effects. The endpoint requires authentication and +// returns cluster health status on success. +// +// Response semantics: 200 means fully valid. 403 signals a recognized token +// blocked by an IP filter, so it still counts as verified. 401 means the +// token is unknown or expired. +func verifyAPIToken(ctx context.Context, client *http.Client, baseURL, token string) (bool, error) { + endpoint, err := url.JoinPath(baseURL, "/api/v1/health-json") + if err != nil { + return false, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return false, err + } + req.Header.Set("Authorization", "Bearer "+token) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusForbidden: + // Token was recognized but blocked (IP filter, permission change). + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} diff --git a/pkg/detectors/humioapitoken/humioapitoken_integration_test.go b/pkg/detectors/humioapitoken/humioapitoken_integration_test.go new file mode 100644 index 000000000000..2076ae7eb183 --- /dev/null +++ b/pkg/detectors/humioapitoken/humioapitoken_integration_test.go @@ -0,0 +1,178 @@ +//go:build detectors +// +build detectors + +package humioapitoken + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +// newTestScanner returns a Scanner with the cloud endpoint configured, mirroring +// what DefaultDetectors() does at startup. Without this, Endpoints() returns +// nothing and the verification loop never runs. +func newTestScanner(client *http.Client) Scanner { + s := Scanner{client: client} + s.SetCloudEndpoint("https://cloud.us.humio.com") + s.UseCloudEndpoint(true) + return s +} + +func TestHumioAPIToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors7") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("HUMIOAPITOKEN") + inactiveSecret := testSecrets.MustGetField("HUMIOAPITOKEN_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: newTestScanner(nil), + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a humio api token %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_HumioAPIToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, real secrets, verification error due to timeout", + s: newTestScanner(common.SaneHttpClientTimeOut(1 * time.Microsecond)), + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a humio api token %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detector_typepb.DetectorType_HumioAPIToken, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: newTestScanner(common.ConstantResponseHttpClient(500, "{}")), + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a humio api token %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detector_typepb.DetectorType_HumioAPIToken, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, unverified", + s: newTestScanner(nil), + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a humio api token %s within but not valid", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_HumioAPIToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: newTestScanner(nil), + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("HumioAPIToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "SecretParts", "ExtraData") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("HumioAPIToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/humioapitoken/humioapitoken_test.go b/pkg/detectors/humioapitoken/humioapitoken_test.go new file mode 100644 index 000000000000..8d01a72b3c6d --- /dev/null +++ b/pkg/detectors/humioapitoken/humioapitoken_test.go @@ -0,0 +1,292 @@ +package humioapitoken + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + // 32-char prefix + tilde + 44-char suffix (repository API token) + validPattern32 = "pHUw1oLASALFmt2ppNvwCR0Meo2nHQ15~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR" + // 24-char prefix + tilde + 44-char suffix (personal API token) + validPattern24 = "abcDefGhiJklMnoPqrStUvWx~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR" +) + +func TestHumioAPIToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - python client config", + input: ` + from humiolib.HumioClient import HumioClient + + client = HumioClient( + base_url="https://cloud.us.humio.com", + api_token="pHUw1oLASALFmt2ppNvwCR0Meo2nHQ15~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR", + ) + results = client.search_repo("sandbox", "loglevel=ERROR")`, + want: []string{validPattern32}, + }, + { + name: "valid pattern - env file with PAT", + input: ` + # LogScale personal access token for CI/CD pipeline + LOGSCALE_API_TOKEN=abcDefGhiJklMnoPqrStUvWx~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR + LOGSCALE_BASE_URL=https://cloud.humio.com + LOG_LEVEL=info`, + want: []string{validPattern24}, + }, + { + name: "valid pattern - logscale keyword in docker compose", + input: ` + services: + log-shipper: + image: fluent/fluent-bit:latest + environment: + - LOGSCALE_TOKEN=pHUw1oLASALFmt2ppNvwCR0Meo2nHQ15~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR + - LOGSCALE_HOST=https://cloud.us.humio.com + restart: always`, + want: []string{validPattern32}, + }, + { + name: "valid pattern - ignore duplicate", + input: fmt.Sprintf("humio token = '%s' | '%s'", validPattern32, validPattern32), + want: []string{validPattern32}, + }, + { + name: "valid pattern - token far from keyword", + input: ` + # Humio repository configuration + # Generated by setup wizard on 2025-01-15 + + + api_token = "pHUw1oLASALFmt2ppNvwCR0Meo2nHQ15~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR"`, + want: []string{validPattern32}, + }, + { + name: "invalid pattern - tilde present but suffix too short", + input: ` + from humiolib.HumioClient import HumioClient + client = HumioClient(api_token="pHUw1oLASALFmt2ppNvwCR0Meo2nHQ15~ECViF0Ce95uIqFGSS")`, + want: []string{}, + }, + { + name: "invalid pattern - tilde present but segments too short", + input: ` + # humio config + token = "abc~def"`, + want: []string{}, + }, + { + name: "invalid pattern - wrong prefix length (28 chars)", + input: ` + # humio config + token = "abcDefGhiJklMnoPqrStUvWxYzAb~ECViF0Ce95uIqFGSSatjKWX71EOzvkpVGSuc3zGYqJgR"`, + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf("expected %d results, got %d", len(test.want), len(results)) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + +// Exercise the verification logic against a mock health-json server. Each +// HTTP status code maps to a specific verified/error outcome — 403 counts as +// verified because the server recognized the token (just blocked by IP filter). +func TestHumioAPIToken_Verification(t *testing.T) { + tests := []struct { + name string + statusCode int + wantVerified bool + wantErr bool + }{ + {"200 - valid token", http.StatusOK, true, false}, + {"401 - unknown token", http.StatusUnauthorized, false, false}, + {"403 - recognized but blocked", http.StatusForbidden, true, false}, + {"500 - server error", http.StatusInternalServerError, false, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.statusCode) + })) + defer ts.Close() + + s := Scanner{} + s.SetCloudEndpoint(ts.URL) + s.UseCloudEndpoint(true) + + input := fmt.Sprintf("humio_token = '%s'", validPattern32) + results, err := s.FromData(context.Background(), true, []byte(input)) + if err != nil { + t.Fatalf("FromData error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + r := results[0] + if r.Verified != tt.wantVerified { + t.Errorf("Verified = %v, want %v", r.Verified, tt.wantVerified) + } + if tt.wantErr && r.VerificationError() == nil { + t.Error("expected verification error, got nil") + } + if !tt.wantErr && r.VerificationError() != nil { + t.Errorf("unexpected verification error: %v", r.VerificationError()) + } + }) + } +} + +// Verify that a successful verification on a later endpoint clears any +// error from an earlier failed attempt (e.g. first endpoint returns 500, +// second returns 200). +func TestHumioAPIToken_Verification_StaleErrorCleared(t *testing.T) { + callCount := 0 + ts500 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + w.WriteHeader(http.StatusInternalServerError) + })) + defer ts500.Close() + + ts200 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + w.WriteHeader(http.StatusOK) + })) + defer ts200.Close() + + s := Scanner{} + _ = s.SetConfiguredEndpoints(ts500.URL, ts200.URL) + + input := fmt.Sprintf("humio_token = '%s'", validPattern32) + results, err := s.FromData(context.Background(), true, []byte(input)) + if err != nil { + t.Fatalf("FromData error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + r := results[0] + if !r.Verified { + t.Error("expected Verified = true after second endpoint succeeded") + } + if r.VerificationError() != nil { + t.Errorf("stale verification error not cleared: %v", r.VerificationError()) + } +} + +// Verify that a clean 401 from the wrong region does not erase a transient +// error from an earlier endpoint. The verification error should survive so +// consumers know the result is uncertain, not definitively "not valid." +func TestHumioAPIToken_Verification_ErrorPreservedAcross401(t *testing.T) { + ts500 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer ts500.Close() + + ts401 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + defer ts401.Close() + + s := Scanner{} + _ = s.SetConfiguredEndpoints(ts500.URL, ts401.URL) + + input := fmt.Sprintf("humio_token = '%s'", validPattern32) + results, err := s.FromData(context.Background(), true, []byte(input)) + if err != nil { + t.Fatalf("FromData error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + r := results[0] + if r.Verified { + t.Error("expected Verified = false") + } + if r.VerificationError() == nil { + t.Error("expected verification error to be preserved after 401 from another endpoint, got nil") + } +} + +func TestHumioAPIToken_TokenType(t *testing.T) { + tests := []struct { + name string + token string + want string + }{ + {"24-char prefix is Personal API Token", validPattern24, "Personal API Token"}, + {"32-char prefix is Repository API Token", validPattern32, "Repository API Token"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := humioTokenType(tt.token) + if got != tt.want { + t.Errorf("humioTokenType() = %q, want %q", got, tt.want) + } + }) + } +} + +// Data without humio/logscale context must not reach this detector — the +// Aho-Corasick pre-filter should reject the chunk. +func TestHumioAPIToken_NoKeywordMatch(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + input := fmt.Sprintf("some_generic_token = '%s'", validPattern32) + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input)) + if len(matchedDetectors) != 0 { + t.Errorf("expected no keyword match for input without humio/logscale context, got %d", len(matchedDetectors)) + } +} diff --git a/pkg/detectors/hunter/hunter.go b/pkg/detectors/hunter/hunter.go index 246d94fdc592..589ff0325bee 100644 --- a/pkg/detectors/hunter/hunter.go +++ b/pkg/detectors/hunter/hunter.go @@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/hybiscus/hybiscus.go b/pkg/detectors/hybiscus/hybiscus.go index d39e970eb8fe..7f7fbd78ead7 100644 --- a/pkg/detectors/hybiscus/hybiscus.go +++ b/pkg/detectors/hybiscus/hybiscus.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-KEY", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/hypertrack/hypertrack.go b/pkg/detectors/hypertrack/hypertrack.go index ab18081fad32..b9095ddb20f5 100644 --- a/pkg/detectors/hypertrack/hypertrack.go +++ b/pkg/detectors/hypertrack/hypertrack.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ibmclouduserkey/ibmclouduserkey.go b/pkg/detectors/ibmclouduserkey/ibmclouduserkey.go index 6367b6061c95..494be3e4256a 100644 --- a/pkg/detectors/ibmclouduserkey/ibmclouduserkey.go +++ b/pkg/detectors/ibmclouduserkey/ibmclouduserkey.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", "Basic Yng6Yng=") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/iconfinder/iconfinder.go b/pkg/detectors/iconfinder/iconfinder.go index d9fd8627b97b..bf805236f176 100644 --- a/pkg/detectors/iconfinder/iconfinder.go +++ b/pkg/detectors/iconfinder/iconfinder.go @@ -3,10 +3,11 @@ package iconfinder import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/iexapis/iexapis.go b/pkg/detectors/iexapis/iexapis.go index 0733fecab5a9..083c35323bc1 100644 --- a/pkg/detectors/iexapis/iexapis.go +++ b/pkg/detectors/iexapis/iexapis.go @@ -2,10 +2,11 @@ package iexapis import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/iexcloud/iexcloud.go b/pkg/detectors/iexcloud/iexcloud.go index 780f3c6fbdbc..c3605aca4d13 100644 --- a/pkg/detectors/iexcloud/iexcloud.go +++ b/pkg/detectors/iexcloud/iexcloud.go @@ -3,10 +3,11 @@ package iexcloud import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/imagekit/imagekit.go b/pkg/detectors/imagekit/imagekit.go index 78899e66bc91..aa253842cf62 100644 --- a/pkg/detectors/imagekit/imagekit.go +++ b/pkg/detectors/imagekit/imagekit.go @@ -2,10 +2,11 @@ package imagekit import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/imagga/imagga.go b/pkg/detectors/imagga/imagga.go index b04d11f4b237..688ab28aa6c6 100644 --- a/pkg/detectors/imagga/imagga.go +++ b/pkg/detectors/imagga/imagga.go @@ -3,10 +3,11 @@ package imagga import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/impala/impala.go b/pkg/detectors/impala/impala.go index 037d133748b9..040419409b72 100644 --- a/pkg/detectors/impala/impala.go +++ b/pkg/detectors/impala/impala.go @@ -2,10 +2,11 @@ package impala import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/infura/infura.go b/pkg/detectors/infura/infura.go index e988aa8e8026..a782bb33a57f 100644 --- a/pkg/detectors/infura/infura.go +++ b/pkg/detectors/infura/infura.go @@ -2,11 +2,12 @@ package infura import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,24 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := strings.NewReader(`{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}`) - req, err := http.NewRequestWithContext(ctx, "POST", "https://mainnet.infura.io/v3/"+resMatch, payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if strings.Contains(body, `"result"`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -72,6 +58,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + payload := strings.NewReader(`{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}`) + req, err := http.NewRequestWithContext(ctx, "POST", "https://mainnet.infura.io/v3/"+resMatch, payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if strings.Contains(body, `"result"`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Infura } diff --git a/pkg/detectors/insightly/insightly.go b/pkg/detectors/insightly/insightly.go index 80619821340c..22c9e77e5f5b 100644 --- a/pkg/detectors/insightly/insightly.go +++ b/pkg/detectors/insightly/insightly.go @@ -2,10 +2,11 @@ package insightly import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/instabot/instabot.go b/pkg/detectors/instabot/instabot.go index 64362555aacb..9d47bec36041 100644 --- a/pkg/detectors/instabot/instabot.go +++ b/pkg/detectors/instabot/instabot.go @@ -2,10 +2,11 @@ package instabot import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Instabot-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/instamojo/instamojo.go b/pkg/detectors/instamojo/instamojo.go index e36ac2c43e76..15203d6094c2 100644 --- a/pkg/detectors/instamojo/instamojo.go +++ b/pkg/detectors/instamojo/instamojo.go @@ -61,30 +61,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if client == nil { client = defaultClient } - payload := strings.NewReader("grant_type=client_credentials&client_id=" + resClientId + "&client_secret=" + resSecret) - - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.instamojo.com/oauth2/token/", payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") { - s1.Verified = true - } else { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resSecret) - } - } else { - s1.SetVerificationError(err, resSecret) - } + isVerified, verificationErr := verifyMatch(ctx, client, resClientId, resSecret) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resClientId, resSecret) } results = append(results, s1) @@ -94,6 +73,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resClientId, resSecret string) (bool, error) { + payload := strings.NewReader("grant_type=client_credentials&client_id=" + resClientId + "&client_secret=" + resSecret) + + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.instamojo.com/oauth2/token/", payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") { + return true, nil + } + + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Instamojo } diff --git a/pkg/detectors/intercom/intercom.go b/pkg/detectors/intercom/intercom.go index cdfbd56c2cf4..5680d5e2c8f1 100644 --- a/pkg/detectors/intercom/intercom.go +++ b/pkg/detectors/intercom/intercom.go @@ -3,10 +3,11 @@ package intercom import ( "context" "encoding/base64" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -63,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/interseller/interseller.go b/pkg/detectors/interseller/interseller.go index 6eee0a98b6ee..ac2c9e38f534 100644 --- a/pkg/detectors/interseller/interseller.go +++ b/pkg/detectors/interseller/interseller.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/intra42/intra42.go b/pkg/detectors/intra42/intra42.go index cc741b10b266..31346cd34c39 100644 --- a/pkg/detectors/intra42/intra42.go +++ b/pkg/detectors/intra42/intra42.go @@ -102,7 +102,7 @@ func verifyIntra42(ctx context.Context, client *http.Client, resMatch string, re if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/intrinio/intrinio.go b/pkg/detectors/intrinio/intrinio.go index 0cf5c3298f41..9a8ba435d531 100644 --- a/pkg/detectors/intrinio/intrinio.go +++ b/pkg/detectors/intrinio/intrinio.go @@ -3,10 +3,11 @@ package intrinio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/invoiceocean/invoiceocean.go b/pkg/detectors/invoiceocean/invoiceocean.go index a0cde788265b..46082090366b 100644 --- a/pkg/detectors/invoiceocean/invoiceocean.go +++ b/pkg/detectors/invoiceocean/invoiceocean.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ip2location/ip2location.go b/pkg/detectors/ip2location/ip2location.go index 8b5adcdf9f3a..f0bc0e607268 100644 --- a/pkg/detectors/ip2location/ip2location.go +++ b/pkg/detectors/ip2location/ip2location.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/ipapi/ipapi.go b/pkg/detectors/ipapi/ipapi.go index 2ba194933eff..76070ce23b87 100644 --- a/pkg/detectors/ipapi/ipapi.go +++ b/pkg/detectors/ipapi/ipapi.go @@ -3,11 +3,12 @@ package ipapi import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyString := string(bodyBytes) valid := strings.Contains(bodyString, "continent_code") || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if valid { s1.Verified = true diff --git a/pkg/detectors/ipgeolocation/ipgeolocation.go b/pkg/detectors/ipgeolocation/ipgeolocation.go index 13fe2ee728cf..52a3883b52ce 100644 --- a/pkg/detectors/ipgeolocation/ipgeolocation.go +++ b/pkg/detectors/ipgeolocation/ipgeolocation.go @@ -3,10 +3,11 @@ package ipgeolocation import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ipinfo/ipinfo.go b/pkg/detectors/ipinfo/ipinfo.go index ec3165606de1..a8d8632a2f2c 100644 --- a/pkg/detectors/ipinfo/ipinfo.go +++ b/pkg/detectors/ipinfo/ipinfo.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_IPInfo, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 403 { diff --git a/pkg/detectors/ipinfo/ipinfo_integration_test.go b/pkg/detectors/ipinfo/ipinfo_integration_test.go index 1d1ccda12f4b..19c3d5ab1d5f 100644 --- a/pkg/detectors/ipinfo/ipinfo_integration_test.go +++ b/pkg/detectors/ipinfo/ipinfo_integration_test.go @@ -133,11 +133,14 @@ func TestIpinfo_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: \n %+v", got[i]) + } if (got[i].VerificationError() != nil) != tt.wantVerificationErr { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Ipinfo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/ipinfodb/ipinfodb.go b/pkg/detectors/ipinfodb/ipinfodb.go index 496dde166573..431f2e2d7297 100644 --- a/pkg/detectors/ipinfodb/ipinfodb.go +++ b/pkg/detectors/ipinfodb/ipinfodb.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"statusCode": "OK"`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/ipstack/ipstack.go b/pkg/detectors/ipstack/ipstack.go index 3d80a33dc324..2f100d48a47c 100644 --- a/pkg/detectors/ipstack/ipstack.go +++ b/pkg/detectors/ipstack/ipstack.go @@ -2,11 +2,12 @@ package ipstack import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -45,25 +46,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ipstack.com/134.201.250.155?access_key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - validResponse := strings.Contains(body, "continent_code") || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - - if validResponse { - s1.Verified = true - } - } - + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -72,6 +57,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ipstack.com/134.201.250.155?access_key="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + validResponse := strings.Contains(body, "continent_code") || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) + + if validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_IpStack } diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go index cb9816f468f4..963c77be3601 100644 --- a/pkg/detectors/jdbc/jdbc.go +++ b/pkg/detectors/jdbc/jdbc.go @@ -5,7 +5,7 @@ import ( "database/sql" "errors" "fmt" - "regexp" + "regexp" //nolint:depguard // used instead of github.com/wasilibs/go-re2 due to differences in utf-8 handling "strings" "time" @@ -16,12 +16,12 @@ import ( type Scanner struct { detectors.DefaultMultiPartCredentialProvider - ignorePatterns []regexp.Regexp + ignorePatterns []*regexp.Regexp } func New(opts ...func(*Scanner)) *Scanner { scanner := &Scanner{ - ignorePatterns: []regexp.Regexp{}, + ignorePatterns: []*regexp.Regexp{}, } for _, opt := range opts { opt(scanner) @@ -32,13 +32,13 @@ func New(opts ...func(*Scanner)) *Scanner { func WithIgnorePattern(ignoreStrings []string) func(*Scanner) { return func(s *Scanner) { - var ignorePatterns []regexp.Regexp + var ignorePatterns []*regexp.Regexp for _, ignoreString := range ignoreStrings { ignorePattern, err := regexp.Compile(ignoreString) if err != nil { panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err)) } - ignorePatterns = append(ignorePatterns, *ignorePattern) + ignorePatterns = append(ignorePatterns, ignorePattern) } s.ignorePatterns = ignorePatterns @@ -83,29 +83,45 @@ matchLoop: DetectorType: detector_typepb.DetectorType_JDBC, Raw: []byte(jdbcConn), Redacted: tryRedactAnonymousJDBC(jdbcConn), + SecretParts: map[string]string{"connection_string": jdbcConn}, } - if verify { - j, err := NewJDBC(logCtx, jdbcConn) - if err != nil { - continue + // Try to parse connection info for ExtraData regardless of verification. + if j, parseErr := NewJDBC(logCtx, jdbcConn); parseErr == nil { + if info := j.GetConnectionInfo(); info != nil { + extraData := make(map[string]string) + if info.Host != "" { + extraData["host"] = info.Host + } + if info.User != "" { + extraData["username"] = info.User + } + if info.Database != "" { + extraData["database"] = info.Database + } + if len(extraData) > 0 { + result.ExtraData = extraData + } } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - pingRes := j.ping(ctx) - result.Verified = pingRes.err == nil - // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the - // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate - // detectors to use tri-state verification. - if pingRes.err != nil && !pingRes.determinate { - err = pingRes.err - result.SetVerificationError(err, jdbcConn) - } - result.SecretParts = map[string]string{ - "connection_string": jdbcConn, + if verify { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + pingRes := j.ping(ctx) + result.Verified = pingRes.err == nil + // If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the + // behavior before tri-state verification was introduced and preserving it allows us to gradually migrate + // detectors to use tri-state verification. + if pingRes.err != nil && !pingRes.determinate { + result.SetVerificationError(pingRes.err, jdbcConn) + } } - // TODO: specialized redaction + } else if verify { + // The connection string could not be parsed, so the credential was never tested. + // That is an indeterminate outcome, not a rejection: we report it as unverified carrying the parse error. + // Dropping it here made the reported findings depend on whether verification was requested, + // since the unparseable match is kept when verify is false. + result.SetVerificationError(parseErr, jdbcConn) } results = append(results, result) @@ -252,7 +268,7 @@ func pingErr(ctx context.Context, driverName, conn string) error { if err != nil { return err } - defer db.Close() + defer func() { _ = db.Close() }() if err := db.PingContext(ctx); err != nil { return err diff --git a/pkg/detectors/jdbc/jdbc_integration_test.go b/pkg/detectors/jdbc/jdbc_integration_test.go index f20b2a1fd6f4..ea2e71d04bba 100644 --- a/pkg/detectors/jdbc/jdbc_integration_test.go +++ b/pkg/detectors/jdbc/jdbc_integration_test.go @@ -195,6 +195,7 @@ func TestJdbcVerified(t *testing.T) { t.Fatal("no raw secret present") } got[i].Raw = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) @@ -323,6 +324,7 @@ func TestJdbc_FromChunk(t *testing.T) { } got[i].Raw = nil got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go index 2ba0c937a95e..267346caf15e 100644 --- a/pkg/detectors/jdbc/jdbc_test.go +++ b/pkg/detectors/jdbc/jdbc_test.go @@ -151,6 +151,76 @@ func TestJdbc_Pattern(t *testing.T) { } } +func TestJdbc_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "mysql with basic auth", + data: `jdbc:mysql://root:password@localhost:3306/testdb`, + wantHost: "tcp(localhost:3306)", + wantUsername: "root", + wantDatabase: "testdb", + }, + { + name: "postgresql with basic auth", + data: `jdbc:postgresql://postgres:secret@dbhost:5432/mydb`, + wantHost: "dbhost:5432", + wantUsername: "postgres", + wantDatabase: "mydb", + }, + { + name: "sqlserver with semicolon params", + data: `jdbc:sqlserver://server.example.com:1433;database=testdb;user=sa;password=Pass123`, + wantHost: "server.example.com:1433", + wantUsername: "sa", + wantDatabase: "testdb", + }, + { + name: "mysql with query params for credentials", + data: `jdbc:mysql://dbhost:3307/testdb?user=admin&password=secret`, + wantHost: "tcp(dbhost:3307)", + wantUsername: "admin", + wantDatabase: "testdb", + }, + { + name: "postgresql with query params for credentials", + data: `jdbc:postgresql://localhost:1521/testdb?sslmode=disable&password=testpassword&user=testuser`, + wantHost: "localhost:1521", + wantUsername: "testuser", + wantDatabase: "testdb", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + }) + } +} + +func TestJdbc_ExtraData_UnsupportedSubprotocol(t *testing.T) { + // For unsupported subprotocols (e.g., sqlite), ExtraData should be nil + // because we can't parse connection info, but the result should still be returned. + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(`jdbc:sqlite:/data/test.db`)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + assert.Nil(t, results[0].ExtraData, "expected nil ExtraData for unsupported subprotocol") +} + func TestJdbc_FromDataWithIgnorePattern(t *testing.T) { type args struct { ctx context.Context diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.go index 6b954ec19dd4..65027e26122e 100644 --- a/pkg/detectors/jiratoken/v1/jiratoken.go +++ b/pkg/detectors/jiratoken/v1/jiratoken.go @@ -8,6 +8,9 @@ import ( "fmt" "io" "net/http" + "net/url" + "slices" + "sort" "strings" regexp "github.com/wasilibs/go-re2" @@ -20,15 +23,25 @@ import ( type Scanner struct { detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter client *http.Client } // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Versioner = (*Scanner)(nil) +var _ detectors.EndpointCustomizer = (*Scanner)(nil) +var _ detectors.CloudProvider = (*Scanner)(nil) func (Scanner) Version() int { return 1 } +// CloudHost is used whenever the scanned data carries no Atlassian host of its own. +// reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943 +// The graphql API answers on this host as long as the credentials are valid. +const CloudHost = "api.atlassian.com" + +func (Scanner) CloudEndpoint() string { return "https://" + CloudHost } + var ( defaultClient = detectors.DetectorHttpClientWithLocalAddresses @@ -40,8 +53,79 @@ var ( invalidHosts = simple.NewCache[struct{}]() errNoHost = errors.New("no such host") + + // atlassianDomains are the only hosts that can answer the verification request. + // domainPat matches any domain-shaped string near an atlassian/jira/confluence keyword, so without + // this filter the detector would send the email and token to unrelated hosts found in the same chunk. + atlassianDomains = []string{"atlassian.net", "atlassian.com", "jira.com"} ) +// IsAtlassianHost reports whether host is owned by Atlassian and can therefore be used for verification. +func IsAtlassianHost(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, domain := range atlassianDomains { + if host == domain || strings.HasSuffix(host, "."+domain) { + return true + } + } + return false +} + +// EndpointHost reduces an endpoint to its bare host. Found domains arrive without a scheme while cloud and +// user-configured endpoints carry one, and the analyzer expects SecretParts["domain"] to be a bare host. +func EndpointHost(endpoint string) string { + endpoint = strings.TrimSuffix(strings.TrimSpace(endpoint), "/") + if u, err := url.Parse(endpoint); err == nil && u.Host != "" { + return u.Host + } + return endpoint +} + +// FoundHosts returns the Atlassian hosts among the domains matched in the data, sorted so results are +// deterministic. domainPat matches any domain-shaped string, so everything else is dropped rather than +// being sent the email and token. +func FoundHosts(domains map[string]struct{}) []string { + hosts := make([]string, 0, len(domains)) + for domain := range domains { + if IsAtlassianHost(domain) { + hosts = append(hosts, domain) + } + } + sort.Strings(hosts) + return hosts +} + +// VerificationHosts reduces endpoints to the bare hosts to verify against, dropping duplicates. The +// analyzer expects SecretParts["domain"] to be a bare host, and a user-configured endpoint can repeat a +// found one. +// +// CloudHost is only a fallback for data that named no Atlassian host, and EndpointSetter places it ahead +// of the found hosts, so it is dropped when the data named hosts of its own. Data that named CloudHost +// itself keeps it, since then it is a found host rather than the fallback. +func VerificationHosts(endpoints, found []string) []string { + seen := make(map[string]struct{}, len(endpoints)) + hosts := make([]string, 0, len(endpoints)) + dropCloud := len(found) > 0 && !slices.Contains(found, CloudHost) + + for _, endpoint := range endpoints { + host := EndpointHost(endpoint) + if host == "" || (dropCloud && host == CloudHost) { + continue + } + if _, ok := seen[host]; ok { + continue + } + seen[host] = struct{}{} + hosts = append(hosts, host) + } + return hosts +} + +// verificationURL builds the graphql URL for a bare host such as "acme.atlassian.net". +func verificationURL(host string) string { + return "https://" + host + "/gateway/api/graphql" +} + type JIRAGraphQLResponse struct { Data struct { Me struct { @@ -83,28 +167,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result uniqueEmails[strings.ToLower(email[1])] = struct{}{} } - if len(uniqueDomains) == 0 { - // reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943 - // In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid - uniqueDomains["api.atlassian.com"] = struct{}{} - } + found := FoundHosts(uniqueDomains) + hosts := VerificationHosts(s.Endpoints(found...), found) for email := range uniqueEmails { for token := range uniqueTokens { - for domain := range uniqueDomains { + for _, domain := range hosts { if invalidHosts.Exists(domain) { - delete(uniqueDomains, domain) continue } s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_JiraToken, Raw: []byte(token), - RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)), + RawV2: fmt.Appendf(nil, "%s:%s:%s", email, token, domain), ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: map[string]string{ + "token": token, + "domain": domain, + "email": email, + }, } if verify { @@ -118,12 +203,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.SetVerificationError(verificationErr, token) } - if isVerified { - s1.SecretParts = map[string]string{ - "token": token, - "domain": domain, - "email": email, - } + + // The credential is the same across endpoints, so once one of them verifies it there + // is nothing to learn from the rest. + if s1.Verified { + results = append(results, s1) + break } } @@ -148,7 +233,7 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to } // api docs: https://developer.atlassian.com/platform/atlassian-graphql-api/graphql/#authentication - req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+domain+"/gateway/api/graphql", bytes.NewBuffer(jsonBody)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, verificationURL(domain), bytes.NewBuffer(jsonBody)) if err != nil { return false, err } @@ -173,18 +258,29 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to }() // the API returns 200 if the token is valid - switch resp.StatusCode { - case http.StatusOK: + switch { + case resp.StatusCode == http.StatusOK: var jiraResp JIRAGraphQLResponse if err := json.NewDecoder(resp.Body).Decode(&jiraResp); err != nil { return false, nil // can't decode response in case of 200 OK = not valid JIRA domain } - return true, nil - case http.StatusUnauthorized: + // A 200 on its own isn't enough - unrelated hosts can also return JSON and get flagged as verified. + // Only trust it when the body actually carries the authenticated user's name. + return jiraResp.Data.Me.User.Name != "", nil + case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden: + // The credentials were rejected. Nothing to retry. return false, nil - default: + case resp.StatusCode == http.StatusTooManyRequests, resp.StatusCode >= 500: + // Rate limiting and server errors are genuinely transient, so keep them indeterminate + // and let the secret be retried. return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + default: + // Anything else (202, 3xx, 404, ...) is not an Atlassian authentication response at all. + // domainPat matches any domain-shaped string near a jira/atlassian/confluence keyword, so this + // request may well have gone to an unrelated host that happily accepts POSTs. Retrying will + // return the same status forever, so treat it as a terminal "not verified" instead of an error. + return false, nil } } diff --git a/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go b/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go index cb97bf9626d2..50cc391faf31 100644 --- a/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go +++ b/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go @@ -118,7 +118,7 @@ func TestJiraToken_FromChunk(t *testing.T) { wantVerificationErr: true, }, { - name: "found, verified but unexpected api surface", + name: "found, unexpected api surface is terminal and not an error", s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, args: args{ ctx: context.Background(), @@ -136,11 +136,14 @@ func TestJiraToken_FromChunk(t *testing.T) { }, }, wantErr: false, - wantVerificationErr: true, + wantVerificationErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.s.SetCloudEndpoint(tt.s.CloudEndpoint()) + tt.s.UseCloudEndpoint(true) + tt.s.UseFoundEndpoints(true) got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr) @@ -154,7 +157,7 @@ func TestJiraToken_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } @@ -165,6 +168,9 @@ func TestJiraToken_FromChunk(t *testing.T) { func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := Scanner{} + s.SetCloudEndpoint(s.CloudEndpoint()) + s.UseCloudEndpoint(true) + s.UseFoundEndpoints(true) for name, data := range detectors.MustGetBenchmarkData() { benchmark.Run(name, func(b *testing.B) { b.ResetTimer() diff --git a/pkg/detectors/jiratoken/v1/jiratoken_test.go b/pkg/detectors/jiratoken/v1/jiratoken_test.go index d3352347d3a0..cba80e7e2af9 100644 --- a/pkg/detectors/jiratoken/v1/jiratoken_test.go +++ b/pkg/detectors/jiratoken/v1/jiratoken_test.go @@ -7,6 +7,7 @@ import ( "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -14,7 +15,9 @@ import ( var ( validTokenPattern = "Z7VoIYJ0K4rFWLBfkhOsLAWX" invalidTokenPattern = "Z7VoI?J0K4rF#LBfkhO&LAWX" - validDomainPattern = "hereisavalidsubdomain.heresalongdomain.com" + validDomainPattern = "hereisavalidsubdomain.atlassian.net" + foreignDomainPattern = "hereisavalidsubdomain.heresalongdomain.com" + cloudDomain = "api.atlassian.com" invalidDomainPattern = "?y4r3fs1ewqec12v1e3tl.5Hcsrcehic89saXd.ro@" validEmailPattern = "xfkf_bz7@grum.com" invalidEmailPattern = "xfKF_BZq7/grum.com" @@ -23,6 +26,9 @@ var ( func TestJiraToken_Pattern(t *testing.T) { d := Scanner{} + d.SetCloudEndpoint(d.CloudEndpoint()) + d.UseCloudEndpoint(true) + d.UseFoundEndpoints(true) ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) tests := []struct { name string @@ -32,7 +38,16 @@ func TestJiraToken_Pattern(t *testing.T) { { name: "valid pattern - with keyword jira", input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, validDomainPattern, keyword, validEmailPattern), - want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern}, + // The tenant host found in the data is used on its own. The cloud endpoint must not shadow it, + // or SecretParts["domain"] would stop being the host the analyzer needs. + want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern}, + }, + { + // A domain that isn't Atlassian-owned can't answer the verification request, so it is dropped + // and only the cloud endpoint remains. + name: "non-atlassian domain falls back to the cloud endpoint", + input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, foreignDomainPattern, keyword, validEmailPattern), + want: []string{validEmailPattern + ":" + validTokenPattern + ":" + cloudDomain}, }, { name: "valid pattern - key out of prefix range", @@ -88,3 +103,188 @@ func TestJiraToken_Pattern(t *testing.T) { }) } } + +func TestIsAtlassianHost(t *testing.T) { + tests := map[string]bool{ + "example.atlassian.net": true, + "atlassian.net": true, + "api.atlassian.com": true, + "acme.jira.com": true, + "EXAMPLE.Atlassian.Net": true, + "example.atlassian.net.": true, + // Suffix lookalikes must not pass, or the filter would be trivially bypassed. + "notatlassian.net": false, + "atlassian.net.evil.com": false, + "example.com": false, + "hereisavalidsubdomain.co.uk": false, + } + + for host, want := range tests { + t.Run(host, func(t *testing.T) { + if got := IsAtlassianHost(host); got != want { + t.Errorf("IsAtlassianHost(%q) = %v, want %v", host, got, want) + } + }) + } +} + +func TestFoundHosts(t *testing.T) { + tests := []struct { + name string + domains []string + want []string + }{ + { + name: "non-atlassian domains are dropped", + domains: []string{"tenant.atlassian.net", "hooks.example.com"}, + want: []string{"tenant.atlassian.net"}, + }, + { + name: "nothing atlassian found", + domains: []string{"hooks.example.com"}, + want: []string{}, + }, + { + // The cloud host is a normal Atlassian host when it shows up in the data. + name: "cloud host in the data is kept", + domains: []string{CloudHost}, + want: []string{CloudHost}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + domains := make(map[string]struct{}, len(test.domains)) + for _, d := range test.domains { + domains[d] = struct{}{} + } + if diff := cmp.Diff(test.want, FoundHosts(domains)); diff != "" { + t.Errorf("FoundHosts() diff: (-want +got)\n%s", diff) + } + }) + } +} + +func TestVerificationHosts(t *testing.T) { + const cloudEndpoint = "https://" + CloudHost + + tests := []struct { + name string + endpoints []string + found []string + want []string + }{ + { + // EndpointSetter puts the cloud endpoint ahead of the found ones, so leaving it in would let it + // win the break-on-verified and report a domain the analyzer cannot use. + name: "cloud dropped when the data named a host", + endpoints: []string{cloudEndpoint, "tenant.atlassian.net"}, + found: []string{"tenant.atlassian.net"}, + want: []string{"tenant.atlassian.net"}, + }, + { + name: "cloud kept when the data named nothing", + endpoints: []string{cloudEndpoint}, + found: nil, + want: []string{CloudHost}, + }, + { + // Dropping the cloud host here would leave no hosts at all and produce no results. + name: "cloud kept when the data named it", + endpoints: []string{cloudEndpoint, CloudHost}, + found: []string{CloudHost}, + want: []string{CloudHost}, + }, + { + name: "configured endpoints are reduced to hosts and deduped", + endpoints: []string{"https://jira.example.com:8443/", cloudEndpoint, "tenant.atlassian.net", "https://tenant.atlassian.net", ""}, + found: []string{"tenant.atlassian.net"}, + want: []string{"jira.example.com:8443", "tenant.atlassian.net"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if diff := cmp.Diff(test.want, VerificationHosts(test.endpoints, test.found)); diff != "" { + t.Errorf("VerificationHosts() diff: (-want +got)\n%s", diff) + } + }) + } +} + +func TestJiraToken_Verify(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantVerified bool + wantErr bool + }{ + { + name: "authenticated user in body is verified", + statusCode: 200, + body: `{"data":{"me":{"user":{"name":"Jira User"}}}}`, + wantVerified: true, + }, + { + // A non-Atlassian host can happily return 200 with unrelated JSON, so the status code alone can't be trusted. + name: "200 without a jira user is not verified", + statusCode: 200, + body: `{"method":"POST","host":"example.com"}`, + wantVerified: false, + }, + { + name: "unauthorized is not verified", + statusCode: 401, + body: `{"code":401,"message":"Unauthorized"}`, + wantVerified: false, + }, + { + name: "forbidden is not verified", + statusCode: 403, + body: `{"code":403,"message":"Forbidden"}`, + wantVerified: false, + }, + { + // Atlassian never answers 202 here. It comes from unrelated hosts that accept the POST, and it + // would repeat on every retry, so it has to be terminal rather than an indeterminate error. + name: "accepted is not verified and not an error", + statusCode: 202, + body: `{"status":"accepted"}`, + wantVerified: false, + }, + { + name: "not found is not verified and not an error", + statusCode: 404, + body: `{"code":404,"message":"Not Found"}`, + wantVerified: false, + }, + { + name: "rate limited is indeterminate", + statusCode: 429, + body: `{"code":429,"message":"Too Many Requests"}`, + wantVerified: false, + wantErr: true, + }, + { + name: "server error is indeterminate", + statusCode: 500, + body: `{"code":500,"message":"Internal Server Error"}`, + wantVerified: false, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := common.ConstantResponseHttpClient(test.statusCode, test.body) + verified, err := VerifyJiraToken(context.Background(), client, "user@example.com", "example.atlassian.net", validTokenPattern) + if (err != nil) != test.wantErr { + t.Fatalf("got error = %v, wantErr %v", err, test.wantErr) + } + if verified != test.wantVerified { + t.Errorf("got verified = %v, want %v", verified, test.wantVerified) + } + }) + } +} diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.go index da7c2f6ef103..90e275d7e984 100644 --- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go +++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go @@ -17,14 +17,19 @@ import ( type Scanner struct { client *http.Client detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter } // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Versioner = (*Scanner)(nil) +var _ detectors.EndpointCustomizer = (*Scanner)(nil) +var _ detectors.CloudProvider = (*Scanner)(nil) func (Scanner) Version() int { return 2 } +func (Scanner) CloudEndpoint() string { return "https://" + v1.CloudHost } + var ( defaultClient = detectors.DetectorHttpClientWithLocalAddresses @@ -66,23 +71,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result uniqueEmails[strings.ToLower(email[1])] = struct{}{} } - if len(uniqueDomains) == 0 { - // reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943 - // In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid - uniqueDomains["api.atlassian.com"] = struct{}{} - } + // domainPat here is not even keyword-gated, so it captures every domain in the chunk. + found := v1.FoundHosts(uniqueDomains) + hosts := v1.VerificationHosts(s.Endpoints(found...), found) for email := range uniqueEmails { for token := range uniqueTokens { - for domain := range uniqueDomains { + for _, domain := range hosts { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_JiraToken, Raw: []byte(token), - RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)), + RawV2: fmt.Appendf(nil, "%s:%s:%s", email, token, domain), ExtraData: map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/", "version": fmt.Sprintf("%d", s.Version()), }, + SecretParts: map[string]string{ + "token": token, + "domain": domain, + "email": email, + }, } if verify { @@ -90,12 +98,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := v1.VerifyJiraToken(ctx, client, email, domain, token) s1.Verified = isVerified s1.SetVerificationError(verificationErr, token) - if isVerified { - s1.SecretParts = map[string]string{ - "token": token, - "domain": domain, - "email": email, - } + + // The credential is the same across endpoints, so once one of them verifies it there + // is nothing to learn from the rest. + if s1.Verified { + results = append(results, s1) + break } } diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go b/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go index d72a7f331170..1dca3b25b5aa 100644 --- a/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go +++ b/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go @@ -141,6 +141,9 @@ func TestJiraToken_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.s.SetCloudEndpoint(tt.s.CloudEndpoint()) + tt.s.UseCloudEndpoint(true) + tt.s.UseFoundEndpoints(true) got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr) @@ -165,6 +168,9 @@ func TestJiraToken_FromChunk(t *testing.T) { func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := Scanner{} + s.SetCloudEndpoint(s.CloudEndpoint()) + s.UseCloudEndpoint(true) + s.UseFoundEndpoints(true) for name, data := range detectors.MustGetBenchmarkData() { benchmark.Run(name, func(b *testing.B) { b.ResetTimer() diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go b/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go index 4249d15b5e18..40c5b8a4217f 100644 --- a/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go +++ b/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go @@ -12,6 +12,9 @@ import ( func TestJiraToken_Pattern(t *testing.T) { d := Scanner{} + d.SetCloudEndpoint(d.CloudEndpoint()) + d.UseCloudEndpoint(true) + d.UseFoundEndpoints(true) ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) tests := []struct { name string diff --git a/pkg/detectors/jotform/jotform.go b/pkg/detectors/jotform/jotform.go index 19954941372b..fb94224f0190 100644 --- a/pkg/detectors/jotform/jotform.go +++ b/pkg/detectors/jotform/jotform.go @@ -2,10 +2,11 @@ package jotform import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/jumpcloud/jumpcloud.go b/pkg/detectors/jumpcloud/jumpcloud.go index 2f43fb336e40..ae6c4e9b7fc9 100644 --- a/pkg/detectors/jumpcloud/jumpcloud.go +++ b/pkg/detectors/jumpcloud/jumpcloud.go @@ -2,10 +2,11 @@ package jumpcloud import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/jupiterone/jupiterone.go b/pkg/detectors/jupiterone/jupiterone.go index 13d647a3ba37..243a56948a00 100644 --- a/pkg/detectors/jupiterone/jupiterone.go +++ b/pkg/detectors/jupiterone/jupiterone.go @@ -68,12 +68,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() - if res.StatusCode == 200 { + defer func() { _ = res.Body.Close() }() + switch res.StatusCode { + case 200: s1.Verified = true - } else if res.StatusCode == 401 { + case 401: // The secret is determinately not verified (nothing to do) - } else { + default: s1.SetVerificationError(fmt.Errorf("unexpected HTTP response status %d", res.StatusCode), resMatch) } } else { diff --git a/pkg/detectors/juro/juro.go b/pkg/detectors/juro/juro.go index ad23a0d570be..676ed0b8e0c6 100644 --- a/pkg/detectors/juro/juro.go +++ b/pkg/detectors/juro/juro.go @@ -2,10 +2,11 @@ package juro import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/jwt/jwt.go b/pkg/detectors/jwt/jwt.go index 827443391545..aabe0e1fc8cb 100644 --- a/pkg/detectors/jwt/jwt.go +++ b/pkg/detectors/jwt/jwt.go @@ -16,6 +16,7 @@ import ( regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -78,6 +79,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result client := detectors.DetectorHttpClientWithNoLocalAddresses seenMatches := make(map[string]struct{}) + skipUnverified := feature.DropUnverifiedJWTResults.Load() for _, matchGroups := range keyPat.FindAllStringSubmatch(string(data), -1) { match := matchGroups[1] @@ -144,6 +146,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.SetVerificationError(verificationErr, match) } + // Remove unverified results from jwt detector output when the "drop-unverified-jwt-results" feature flag is enabled. + if skipUnverified && verify && !s1.Verified && s1.VerificationError() == nil { + continue + } results = append(results, s1) } @@ -223,7 +229,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pa if err != nil { return false, fmt.Errorf("failed to perform OIDC discovery: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { return false, fmt.Errorf("bad status for OIDC discovery document: %v", resp.Status) } @@ -250,7 +256,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pa if err != nil { return false, fmt.Errorf("failed to fetch JWKS: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { return false, fmt.Errorf("bad status for JWKS: %v", resp.Status) } diff --git a/pkg/detectors/jwt/jwt_test.go b/pkg/detectors/jwt/jwt_test.go index ede498035f9a..fd4b509e6a41 100644 --- a/pkg/detectors/jwt/jwt_test.go +++ b/pkg/detectors/jwt/jwt_test.go @@ -9,6 +9,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" ) // This tests the JWT detector for a number of different cases (mostly HMAC-based) without verification enabled. @@ -135,3 +136,69 @@ func TestJwt_Pattern(t *testing.T) { }) } } + +func TestJwt_SkipUnverified(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + skipUnverified bool + }{ + { + name: "expired token - results should be skipped when skipUnverified is true", + input: "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwNDA2NzIwMCwiZXhwIjoxNzA0MDcwODAwfQ.9z15Ku3lWBC8rFPlvxkX3wx6wRalhoN7PlBRtE7fUXyoHqVLiZG2l5eMo6KwBlAu1L2hFtkvQwiJ79G5o3a4Bw", + want: []string{}, + skipUnverified: true, + }, + { + name: "expired token - results should not be skipped when skipUnverified is false", + input: "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwNDA2NzIwMCwiZXhwIjoxNzA0MDcwODAwfQ.9z15Ku3lWBC8rFPlvxkX3wx6wRalhoN7PlBRtE7fUXyoHqVLiZG2l5eMo6KwBlAu1L2hFtkvQwiJ79G5o3a4Bw", + want: []string{"eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTcwNDA2NzIwMCwiZXhwIjoxNzA0MDcwODAwfQ.9z15Ku3lWBC8rFPlvxkX3wx6wRalhoN7PlBRtE7fUXyoHqVLiZG2l5eMo6KwBlAu1L2hFtkvQwiJ79G5o3a4Bw"}, + skipUnverified: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords()) + return + } + + if test.skipUnverified { + feature.DropUnverifiedJWTResults.Store(true) + } else { + feature.DropUnverifiedJWTResults.Store(false) + } + + results, err := d.FromData(context.Background(), true, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results)) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/kanban/kanban.go b/pkg/detectors/kanban/kanban.go index 6c7a9f3a9e04..d50f30f10cae 100644 --- a/pkg/detectors/kanban/kanban.go +++ b/pkg/detectors/kanban/kanban.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/kanbantool/kanbantool.go b/pkg/detectors/kanbantool/kanbantool.go index d187081aa820..8854f053291e 100644 --- a/pkg/detectors/kanbantool/kanbantool.go +++ b/pkg/detectors/kanbantool/kanbantool.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/karmacrm/karmacrm.go b/pkg/detectors/karmacrm/karmacrm.go index ebeeefb11aab..17d6aead8de6 100644 --- a/pkg/detectors/karmacrm/karmacrm.go +++ b/pkg/detectors/karmacrm/karmacrm.go @@ -3,10 +3,11 @@ package karmacrm import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/keenio/keenio.go b/pkg/detectors/keenio/keenio.go index 2b72e1b63dfb..6080a4979630 100644 --- a/pkg/detectors/keenio/keenio.go +++ b/pkg/detectors/keenio/keenio.go @@ -2,10 +2,11 @@ package keenio import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/kickbox/kickbox.go b/pkg/detectors/kickbox/kickbox.go index cb38ee4715da..9dc64e540ef9 100644 --- a/pkg/detectors/kickbox/kickbox.go +++ b/pkg/detectors/kickbox/kickbox.go @@ -2,10 +2,11 @@ package kickbox import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/klaviyo/klaviyo.go b/pkg/detectors/klaviyo/klaviyo.go index e31d0d2ed1fb..47f232e69cbd 100644 --- a/pkg/detectors/klaviyo/klaviyo.go +++ b/pkg/detectors/klaviyo/klaviyo.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -23,7 +24,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(pk_[[:alnum:]]{34})\b`) + keyPat = regexp.MustCompile(`\b(pk_([0-9a-f]{34}|[A-Za-z0-9]{6}_[0-9a-f]{34}))\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -73,7 +74,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Authorization", "Klaviyo-API-Key "+resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 403 { diff --git a/pkg/detectors/klaviyo/klaviyo_test.go b/pkg/detectors/klaviyo/klaviyo_test.go index e076e06f6e6d..eb3e86fdb03a 100644 --- a/pkg/detectors/klaviyo/klaviyo_test.go +++ b/pkg/detectors/klaviyo/klaviyo_test.go @@ -1,3 +1,4 @@ +// Pattern tests for the Klaviyo detector. package klaviyo import ( @@ -5,77 +6,80 @@ import ( "fmt" "testing" - "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/suite" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -var ( - validPattern = "pk_1234567890abcdefghijklmnopqrstuvwx" - invalidPattern = "pk_1234567890abcdefghijklmnopqrstu-_=" - keyword = "klaviyo" -) +// KlaviyoPatternSuite drives keyPat through the engine's real path (keyword +// prefilter, then FromData). One method per case, so a failure names it. +type KlaviyoPatternSuite struct { + suite.Suite + d Scanner + core *ahocorasick.Core +} -func TestKlaviyo_Pattern(t *testing.T) { - d := Scanner{} - ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - tests := []struct { - name string - input string - want []string - }{ - { - name: "valid pattern - with keyword klaviyo", - input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), - want: []string{validPattern}, - }, - { - name: "invalid pattern", - input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), - want: []string{}, - }, - } +func TestKlaviyoPatternSuite(t *testing.T) { + suite.Run(t, new(KlaviyoPatternSuite)) +} - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { - t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) - return - } +// Fresh scanner and prefilter per method, so state can't leak between cases. +func (s *KlaviyoPatternSuite) SetupTest() { + s.d = Scanner{} + s.core = ahocorasick.NewAhoCorasickCore([]detectors.Detector{s.d}) +} - results, err := d.FromData(context.Background(), false, []byte(test.input)) - if err != nil { - t.Errorf("error = %v", err) - return - } +// Legacy keys must keep matching despite the tighter lowercase-hex rule. +func (s *KlaviyoPatternSuite) TestLegacyHexFormat() { + key := "pk_a1b2c3d4e5f60718293a4b5c6d7e8f9012" // pk_ + 34 lowercase hex + input := fmt.Sprintf("klaviyo token = '%s'", key) + s.Require().NotEmpty(s.core.FindDetectorMatches([]byte(input)), "keyword prefilter should fire") - if len(results) != len(test.want) { - if len(results) == 0 { - t.Errorf("did not receive result") - } else { - t.Errorf("expected %d results, only received %d", len(test.want), len(results)) - } - return - } + results, err := s.d.FromData(context.Background(), false, []byte(input)) + s.Require().NoError(err) + s.Require().Len(results, 1) + s.Equal(key, string(results[0].Raw)) +} + +// The newer prefixed format. Mixed-case company_id is the bit a hex-only +// guess would miss. +func (s *KlaviyoPatternSuite) TestNewPrefixedFormat() { + key := "pk_AbCdEf_f0e1d2c3b4a5968778695a4b3c2d1e0f00" // pk_ + 6 alnum company_id + _ + 34 hex + input := fmt.Sprintf("klaviyo token = '%s'", key) + s.Require().NotEmpty(s.core.FindDetectorMatches([]byte(input)), "keyword prefilter should fire") - actual := make(map[string]struct{}, len(results)) - for _, r := range results { - if len(r.RawV2) > 0 { - actual[string(r.RawV2)] = struct{}{} - } else { - actual[string(r.Raw)] = struct{}{} - } - } - expected := make(map[string]struct{}, len(test.want)) - for _, v := range test.want { - expected[v] = struct{}{} - } + results, err := s.d.FromData(context.Background(), false, []byte(input)) + s.Require().NoError(err) + s.Require().Len(results, 1) + s.Equal(key, string(results[0].Raw)) +} + +// Punctuation in the body should never read as a key. +func (s *KlaviyoPatternSuite) TestPunctuationBodyRejected() { + results, err := s.d.FromData(context.Background(), false, []byte("klaviyo = 'pk_1234567890abcdefghijklmnopqrstu-_='")) + s.Require().NoError(err) + s.Empty(results) +} + +// Legacy bodies are lowercase hex (vendor-confirmed), so uppercase is a false +// positive. +func (s *KlaviyoPatternSuite) TestUppercaseHexRejected() { + results, err := s.d.FromData(context.Background(), false, []byte("klaviyo = 'pk_A1B2C3D4E5F60718293A4B5C6D7E8F9012'")) + s.Require().NoError(err) + s.Empty(results) +} + +// Guards the legacy {34} length. +func (s *KlaviyoPatternSuite) TestLegacyTooShortRejected() { + results, err := s.d.FromData(context.Background(), false, []byte("klaviyo = 'pk_a1b2c3d4e5f60718293a4b5c6d7e8f901'")) // 33 hex + s.Require().NoError(err) + s.Empty(results) +} - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) - } - }) - } +// Guards the new-format {34} tail and trailing \b. +func (s *KlaviyoPatternSuite) TestNewFormatTailTooLongRejected() { + results, err := s.d.FromData(context.Background(), false, []byte("klaviyo = 'pk_AbCdEf_f0e1d2c3b4a5968778695a4b3c2d1e0f001'")) // 35 hex tail + s.Require().NoError(err) + s.Empty(results) } diff --git a/pkg/detectors/klipfolio/klipfolio.go b/pkg/detectors/klipfolio/klipfolio.go index 2a66e3696333..28ce4746b46e 100644 --- a/pkg/detectors/klipfolio/klipfolio.go +++ b/pkg/detectors/klipfolio/klipfolio.go @@ -2,10 +2,11 @@ package klipfolio import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("kf-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/knapsackpro/knapsackpro.go b/pkg/detectors/knapsackpro/knapsackpro.go index b0254ce9841e..0882c9d2db16 100644 --- a/pkg/detectors/knapsackpro/knapsackpro.go +++ b/pkg/detectors/knapsackpro/knapsackpro.go @@ -2,10 +2,11 @@ package knapsackpro import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/kraken/kraken.go b/pkg/detectors/kraken/kraken.go index ac78ccc891cb..a037d53ff7cd 100644 --- a/pkg/detectors/kraken/kraken.go +++ b/pkg/detectors/kraken/kraken.go @@ -86,7 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() out, _ := io.ReadAll(res.Body) if !strings.Contains(string(out), "Invalid key") { s1.Verified = true diff --git a/pkg/detectors/kucoin/kucoin.go b/pkg/detectors/kucoin/kucoin.go index 743788d94638..7e4b24b51464 100644 --- a/pkg/detectors/kucoin/kucoin.go +++ b/pkg/detectors/kucoin/kucoin.go @@ -5,12 +5,13 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/base64" - regexp "github.com/wasilibs/go-re2" "net/http" "strconv" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -88,7 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/kylas/kylas.go b/pkg/detectors/kylas/kylas.go index a94fd737675e..d7df79dbf1c7 100644 --- a/pkg/detectors/kylas/kylas.go +++ b/pkg/detectors/kylas/kylas.go @@ -2,10 +2,11 @@ package kylas import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/languagelayer/languagelayer.go b/pkg/detectors/languagelayer/languagelayer.go index 135edd961e90..c271921e3040 100644 --- a/pkg/detectors/languagelayer/languagelayer.go +++ b/pkg/detectors/languagelayer/languagelayer.go @@ -3,11 +3,12 @@ package languagelayer import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `results`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/larksuiteapikey/larksuiteapikey.go b/pkg/detectors/larksuiteapikey/larksuiteapikey.go index 3a8d9e6a6e32..5aefe72cc12c 100644 --- a/pkg/detectors/larksuiteapikey/larksuiteapikey.go +++ b/pkg/detectors/larksuiteapikey/larksuiteapikey.go @@ -116,7 +116,7 @@ func verifyCredentials(ctx context.Context, client *http.Client, appId, appSecre if bodyResponse.Code == 0 { return true, nil } else { - return false, fmt.Errorf("Verification failed code %d, message %s", bodyResponse.Code, bodyResponse.Message) + return false, fmt.Errorf("verification failed code %d, message %s", bodyResponse.Code, bodyResponse.Message) } } default: diff --git a/pkg/detectors/launchdarkly/launchdarkly.go b/pkg/detectors/launchdarkly/launchdarkly.go index 4ecfca8373c2..7f55d68d3adb 100644 --- a/pkg/detectors/launchdarkly/launchdarkly.go +++ b/pkg/detectors/launchdarkly/launchdarkly.go @@ -72,6 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_LaunchDarkly, Raw: []byte(resMatch), ExtraData: make(map[string]string), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -79,13 +80,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.SetVerificationError(verificationErr) s1.ExtraData = extraData - - // only api keys can be analyzed - if strings.HasPrefix(resMatch, "api-") { - s1.SecretParts = map[string]string{ - "key": resMatch, - } - } } results = append(results, s1) diff --git a/pkg/detectors/leadfeeder/leadfeeder.go b/pkg/detectors/leadfeeder/leadfeeder.go index f619c711f7c9..794cb031c506 100644 --- a/pkg/detectors/leadfeeder/leadfeeder.go +++ b/pkg/detectors/leadfeeder/leadfeeder.go @@ -3,10 +3,11 @@ package leadfeeder import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token token=%s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/lemlist/lemlist.go b/pkg/detectors/lemlist/lemlist.go index c8d779ebb3cc..05b1c785a3f6 100644 --- a/pkg/detectors/lemlist/lemlist.go +++ b/pkg/detectors/lemlist/lemlist.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/lemonsqueezy/lemonsqueezy.go b/pkg/detectors/lemonsqueezy/lemonsqueezy.go index d70476446960..31034a15c5de 100644 --- a/pkg/detectors/lemonsqueezy/lemonsqueezy.go +++ b/pkg/detectors/lemonsqueezy/lemonsqueezy.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/lendflow/lendflow.go b/pkg/detectors/lendflow/lendflow.go index 1723957fd1a9..5585f831d69c 100644 --- a/pkg/detectors/lendflow/lendflow.go +++ b/pkg/detectors/lendflow/lendflow.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Accept", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go b/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go index 9232d985f2a0..bf69f7aebf26 100644 --- a/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go +++ b/pkg/detectors/lessannoyingcrm/lessannoyingcrm.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Accept", "application/vnd.lessannoyingcrm+json; version=3") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/lexigram/lexigram.go b/pkg/detectors/lexigram/lexigram.go index c72b6734be70..6e29f16fe816 100644 --- a/pkg/detectors/lexigram/lexigram.go +++ b/pkg/detectors/lexigram/lexigram.go @@ -3,10 +3,11 @@ package lexigram import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/linearapi/linearapi.go b/pkg/detectors/linearapi/linearapi.go index f6ae661f7381..9b287f1bc6a1 100644 --- a/pkg/detectors/linearapi/linearapi.go +++ b/pkg/detectors/linearapi/linearapi.go @@ -2,11 +2,12 @@ package linearapi import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,26 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - reqDataQuery := strings.NewReader(`{ "query": "{ viewer { id name } }" }`) - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.linear.app/graphql", reqDataQuery) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", resMatch) - - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"data":`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -74,6 +58,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + reqDataQuery := strings.NewReader(`{ "query": "{ viewer { id name } }" }`) + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.linear.app/graphql", reqDataQuery) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", resMatch) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"data":`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_LinearAPI } diff --git a/pkg/detectors/linemessaging/linemessaging.go b/pkg/detectors/linemessaging/linemessaging.go index 9d5d0ea9f6a1..1febd9d64401 100644 --- a/pkg/detectors/linemessaging/linemessaging.go +++ b/pkg/detectors/linemessaging/linemessaging.go @@ -3,10 +3,11 @@ package linemessaging import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/linenotify/linenotify.go b/pkg/detectors/linenotify/linenotify.go index 9688eb91848f..00af92baafce 100644 --- a/pkg/detectors/linenotify/linenotify.go +++ b/pkg/detectors/linenotify/linenotify.go @@ -3,10 +3,11 @@ package linenotify import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/linkpreview/linkpreview.go b/pkg/detectors/linkpreview/linkpreview.go index f2118c5ecc86..dce295e9d6bd 100644 --- a/pkg/detectors/linkpreview/linkpreview.go +++ b/pkg/detectors/linkpreview/linkpreview.go @@ -3,10 +3,11 @@ package linkpreview import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -50,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/liveagent/liveagent.go b/pkg/detectors/liveagent/liveagent.go index 09628f2ec7b8..2d110628e332 100644 --- a/pkg/detectors/liveagent/liveagent.go +++ b/pkg/detectors/liveagent/liveagent.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("apikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 403 { diff --git a/pkg/detectors/livestorm/livestorm.go b/pkg/detectors/livestorm/livestorm.go index e16bf9645ce5..58704ed7149f 100644 --- a/pkg/detectors/livestorm/livestorm.go +++ b/pkg/detectors/livestorm/livestorm.go @@ -2,10 +2,11 @@ package livestorm import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/loadmill/loadmill.go b/pkg/detectors/loadmill/loadmill.go index 826465711f46..902f826e02db 100644 --- a/pkg/detectors/loadmill/loadmill.go +++ b/pkg/detectors/loadmill/loadmill.go @@ -3,10 +3,11 @@ package loadmill import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Loadmill, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/lob/lob.go b/pkg/detectors/lob/lob.go index c86abb288247..755bc79f4faf 100644 --- a/pkg/detectors/lob/lob.go +++ b/pkg/detectors/lob/lob.go @@ -2,31 +2,45 @@ package lob import ( "context" - regexp "github.com/wasilibs/go-re2" + "encoding/json" + "errors" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + defaultClient = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lob"}) + `\b([a-zA-Z0-9_]{40})\b`) + // Secret keys are a live_/test_ prefix followed by 35 lowercase hex characters. Publishable keys + // ((live|test)_pub_ + 31 hex) are excluded, as they are intended to be embedded in client-side code. + keyPat = regexp.MustCompile(`\b((live|test)_[a-f0-9]{35})\b`) ) +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"lob"} + return []string{"live_", "test_"} } // FromData will find and optionally verify Lob secrets in a given set of bytes. @@ -35,28 +49,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result matches := keyPat.FindAllStringSubmatch(dataStr, -1) + uniqueMatches := make(map[string]struct{}) for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) + uniqueMatches[strings.TrimSpace(match[1])] = struct{}{} + } + for resMatch := range uniqueMatches { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Lob, Raw: []byte(resMatch), SecretParts: map[string]string{"key": resMatch}, + ExtraData: map[string]string{ + "environment": resMatch[:4], // live or test + }, } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.lob.com/v1/addresses", nil) - if err != nil { - continue - } - req.SetBasicAuth(resMatch, "") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + verified, err := s.verify(ctx, resMatch) + s1.Verified = verified + s1.SetVerificationError(err) } results = append(results, s1) @@ -65,6 +76,47 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func (s Scanner) verify(ctx context.Context, key string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.lob.com/v1/us_verifications", nil) + if err != nil { + return false, err + } + req.SetBasicAuth(key, "") + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + switch res.StatusCode { + case http.StatusUnprocessableEntity: + // 422 indicates key is active but request body is invalid + return true, nil + case http.StatusForbidden: + var body struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.NewDecoder(res.Body).Decode(&body); err != nil { + return false, errors.New("invalid API response") + } + switch body.Error.Code { + case "billing_address_required", "feature_limit_reached": + // The key authenticated and it is the account, not the key, that is rejected. + return true, nil + case "invalid_api_key": + return false, nil + default: + return false, fmt.Errorf("unexpected error code: <%s> in the API response body", body.Error.Code) + } + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Lob } diff --git a/pkg/detectors/lob/lob_integration_test.go b/pkg/detectors/lob/lob_integration_test.go index 97180296a14b..39204b4aeb31 100644 --- a/pkg/detectors/lob/lob_integration_test.go +++ b/pkg/detectors/lob/lob_integration_test.go @@ -50,6 +50,9 @@ func TestLob_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_Lob, Verified: true, + ExtraData: map[string]string{ + "environment": "live", + }, }, }, wantErr: false, @@ -66,6 +69,9 @@ func TestLob_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_Lob, Verified: false, + ExtraData: map[string]string{ + "environment": "live", + }, }, }, wantErr: false, @@ -95,6 +101,10 @@ func TestLob_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } got[i].Raw = nil + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: \n %+v", got[i]) + } + got[i].SecretParts = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Lob.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/lob/lob_test.go b/pkg/detectors/lob/lob_test.go index 3483cb6f7ab0..9c128084adf7 100644 --- a/pkg/detectors/lob/lob_test.go +++ b/pkg/detectors/lob/lob_test.go @@ -3,6 +3,9 @@ package lob import ( "context" "fmt" + "io" + "net/http" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -12,9 +15,10 @@ import ( ) var ( - validPattern = "XUTbwiuF1_qmP5BvXi4hMeDafM2VoNz5yMH__rI5" - invalidPattern = "XUTbwiuF1_qmP5BvXi4hMeDafM2VoNz5yMH__rI" - keyword = "lob" + validPattern = "live_0979969b3f6cc23ed67e9b650bfaf64f710" + validPatternTest = "test_0979969b3f6cc23ed67e9b650bfaf64f710" + invalidPattern = "live_0979969b3f6cc23ed67e9b650bfaf64f71" + publishablePattern = "live_pub_0979969b3f6cc23ed67e9b650bfaf64" ) func TestLob_Pattern(t *testing.T) { @@ -26,23 +30,38 @@ func TestLob_Pattern(t *testing.T) { want []string }{ { - name: "valid pattern - with keyword lob", - input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), + name: "valid live pattern", + input: fmt.Sprintf("token = '%s'", validPattern), want: []string{validPattern}, }, + { + name: "valid test pattern", + input: fmt.Sprintf("token = '%s'", validPatternTest), + want: []string{validPatternTest}, + }, { name: "valid pattern - ignore duplicate", - input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), + input: fmt.Sprintf("token = '%s' | '%s'", validPattern, validPattern), want: []string{validPattern}, }, { - name: "valid pattern - key out of prefix range", - input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern), + name: "invalid pattern", + input: fmt.Sprintf("'%s'", invalidPattern), want: []string{}, }, { - name: "invalid pattern", - input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), + name: "snake_case identifier of key length", + input: "def test_calculate_total_price_with_tax_rate():", + want: []string{}, + }, + { + name: "camelCase identifier of key length", + input: "func test_someVeryLongCamelCaseFunctionNameXy() {}", + want: []string{}, + }, + { + name: "publishable key", + input: fmt.Sprintf("token = '%s'", publishablePattern), want: []string{}, }, } @@ -89,3 +108,100 @@ func TestLob_Pattern(t *testing.T) { }) } } + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestLob_Verify(t *testing.T) { + lobError := func(code string) string { + return fmt.Sprintf(`{"error":{"message":"...","status_code":403,"code":%q}}`, code) + } + + tests := []struct { + name string + status int + contentType string + body string + want bool + wantErr bool + }{ + { + name: "active key", + status: http.StatusUnprocessableEntity, + body: `{"error":{"message":"...","status_code":422,"code":"invalid"}}`, + want: true, + }, + { + name: "invalid key", + status: http.StatusUnauthorized, + body: `{"error":{"message":"...","status_code":401,"code":"invalid_api_key"}}`, + want: false, + }, + { + name: "active key without a billing address", + status: http.StatusForbidden, + body: lobError("billing_address_required"), + want: true, + }, + { + name: "active key past its free request quota", + status: http.StatusForbidden, + body: lobError("feature_limit_reached"), + want: true, + }, + { + name: "invalid key reported as forbidden", + status: http.StatusForbidden, + body: lobError("invalid_api_key"), + want: false, + }, + { + name: "unrecognized error code", + status: http.StatusForbidden, + body: lobError("some_future_code"), + wantErr: true, + }, + { + name: "forbidden without an API response", + status: http.StatusForbidden, + contentType: "text/html; charset=UTF-8", + body: "Attention Required!", + wantErr: true, + }, + { + name: "unexpected status", + status: http.StatusInternalServerError, + body: `{"error":{"message":"...","status_code":500,"code":"internal_server_error"}}`, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := Scanner{client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + username, _, ok := req.BasicAuth() + if !ok || username != validPattern { + t.Errorf("basic auth username = %q (ok=%t), want %q", username, ok, validPattern) + } + contentType := test.contentType + if contentType == "" { + contentType = "application/json; charset=utf-8" + } + return &http.Response{ + StatusCode: test.status, + Header: http.Header{"Content-Type": []string{contentType}}, + Body: io.NopCloser(strings.NewReader(test.body)), + }, nil + })}} + + got, err := s.verify(context.Background(), validPattern) + if (err != nil) != test.wantErr { + t.Fatalf("error = %v, wantErr %t", err, test.wantErr) + } + if got != test.want { + t.Errorf("verified = %t, want %t", got, test.want) + } + }) + } +} diff --git a/pkg/detectors/locationiq/locationiq.go b/pkg/detectors/locationiq/locationiq.go index 17e2ddbb79ec..8935b8967bc8 100644 --- a/pkg/detectors/locationiq/locationiq.go +++ b/pkg/detectors/locationiq/locationiq.go @@ -2,10 +2,11 @@ package locationiq import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/loggly/loggly.go b/pkg/detectors/loggly/loggly.go index 463c7fa76e62..006ce1136454 100644 --- a/pkg/detectors/loggly/loggly.go +++ b/pkg/detectors/loggly/loggly.go @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/loginradius/loginradius.go b/pkg/detectors/loginradius/loginradius.go index b357b0211cc1..eb3394248a2d 100644 --- a/pkg/detectors/loginradius/loginradius.go +++ b/pkg/detectors/loginradius/loginradius.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/logzio/logzio.go b/pkg/detectors/logzio/logzio.go index 265dd6918d27..50b7d3d83259 100644 --- a/pkg/detectors/logzio/logzio.go +++ b/pkg/detectors/logzio/logzio.go @@ -3,10 +3,11 @@ package logzio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json; charset=utf-8") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/loyverse/loyverse.go b/pkg/detectors/loyverse/loyverse.go index 1434256dcecf..94a88229e34f 100644 --- a/pkg/detectors/loyverse/loyverse.go +++ b/pkg/detectors/loyverse/loyverse.go @@ -3,10 +3,11 @@ package loyverse import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/lunchmoney/lunchmoney.go b/pkg/detectors/lunchmoney/lunchmoney.go index 7baf1387601d..e26faef5f322 100644 --- a/pkg/detectors/lunchmoney/lunchmoney.go +++ b/pkg/detectors/lunchmoney/lunchmoney.go @@ -3,10 +3,11 @@ package lunchmoney import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/luno/luno.go b/pkg/detectors/luno/luno.go index 262c5db1934e..03c2ec50c48e 100644 --- a/pkg/detectors/luno/luno.go +++ b/pkg/detectors/luno/luno.go @@ -2,10 +2,11 @@ package luno import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/m3o/m3o.go b/pkg/detectors/m3o/m3o.go index a510e32d9d30..90509c6cd7d3 100644 --- a/pkg/detectors/m3o/m3o.go +++ b/pkg/detectors/m3o/m3o.go @@ -3,10 +3,11 @@ package m3o import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/madkudu/madkudu.go b/pkg/detectors/madkudu/madkudu.go index 8dc391d30163..97570bb0a5bf 100644 --- a/pkg/detectors/madkudu/madkudu.go +++ b/pkg/detectors/madkudu/madkudu.go @@ -2,10 +2,11 @@ package madkudu import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/magicbell/magicbell.go b/pkg/detectors/magicbell/magicbell.go index 6252058aa9a1..4bfb6870315f 100644 --- a/pkg/detectors/magicbell/magicbell.go +++ b/pkg/detectors/magicbell/magicbell.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-MAGICBELL-USER-EMAIL", emailMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/magnetic/magnetic.go b/pkg/detectors/magnetic/magnetic.go index f25e63c23418..4dd370e99fe5 100644 --- a/pkg/detectors/magnetic/magnetic.go +++ b/pkg/detectors/magnetic/magnetic.go @@ -2,10 +2,11 @@ package magnetic import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mailboxlayer/mailboxlayer.go b/pkg/detectors/mailboxlayer/mailboxlayer.go index a7b48ea91006..9e038a8a79c2 100644 --- a/pkg/detectors/mailboxlayer/mailboxlayer.go +++ b/pkg/detectors/mailboxlayer/mailboxlayer.go @@ -47,29 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://apilayer.net/api/check?access_key="+resMatch+"&email=support@email.com", nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - validResponse := strings.Contains(body, `email`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - - // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} - // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} - // ingenious! - - if validResponse { - s1.Verified = true - } - - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -78,6 +58,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://apilayer.net/api/check?access_key="+resMatch+"&email=support@email.com", nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + validResponse := strings.Contains(body, `email`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) + + // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} + // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} + // ingenious! + + if validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Mailboxlayer } diff --git a/pkg/detectors/mailchimp/mailchimp.go b/pkg/detectors/mailchimp/mailchimp.go index 84f862ce780c..f2b16e0439a6 100644 --- a/pkg/detectors/mailchimp/mailchimp.go +++ b/pkg/detectors/mailchimp/mailchimp.go @@ -42,6 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result result := detectors.Result{ DetectorType: detector_typepb.DetectorType_Mailchimp, Raw: []byte(match), + SecretParts: map[string]string{"key": match}, } result.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/mailchimp/", @@ -59,14 +60,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("accept", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { result.Verified = true } } - result.SecretParts = map[string]string{ - "key": match, - } } results = append(results, result) diff --git a/pkg/detectors/mailerlite/mailerlite.go b/pkg/detectors/mailerlite/mailerlite.go index c6facbbcd944..1b2019b21d93 100644 --- a/pkg/detectors/mailerlite/mailerlite.go +++ b/pkg/detectors/mailerlite/mailerlite.go @@ -2,10 +2,11 @@ package mailerlite import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-MailerLite-ApiKey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mailgun/mailgun.go b/pkg/detectors/mailgun/mailgun.go index 0f6cc497e321..47359d7d46f9 100644 --- a/pkg/detectors/mailgun/mailgun.go +++ b/pkg/detectors/mailgun/mailgun.go @@ -103,7 +103,8 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, _ = res.Body.Close() }() - if res.StatusCode == http.StatusOK { + switch res.StatusCode { + case http.StatusOK: var domains domainResponse if err := json.NewDecoder(res.Body).Decode(&domains); err != nil { return false, nil, fmt.Errorf("error decoding response body: %w", err) @@ -132,9 +133,9 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, } return true, extraData, nil - } else if res.StatusCode == http.StatusUnauthorized { + case http.StatusUnauthorized: return false, nil, nil - } else { + default: return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } } diff --git a/pkg/detectors/mailjetbasicauth/mailjetbasicauth.go b/pkg/detectors/mailjetbasicauth/mailjetbasicauth.go index 09a66a2e97c5..53b73ba46c84 100644 --- a/pkg/detectors/mailjetbasicauth/mailjetbasicauth.go +++ b/pkg/detectors/mailjetbasicauth/mailjetbasicauth.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mailjetsms/mailjetsms.go b/pkg/detectors/mailjetsms/mailjetsms.go index 871a5a68cd07..e88500d01eed 100644 --- a/pkg/detectors/mailjetsms/mailjetsms.go +++ b/pkg/detectors/mailjetsms/mailjetsms.go @@ -3,10 +3,11 @@ package mailjetsms import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mailmodo/mailmodo.go b/pkg/detectors/mailmodo/mailmodo.go index f3781d60288a..eeebb0971c99 100644 --- a/pkg/detectors/mailmodo/mailmodo.go +++ b/pkg/detectors/mailmodo/mailmodo.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("mmApiKey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mailsac/mailsac.go b/pkg/detectors/mailsac/mailsac.go index ad6ff4010cd8..733afaefa9ef 100644 --- a/pkg/detectors/mailsac/mailsac.go +++ b/pkg/detectors/mailsac/mailsac.go @@ -2,10 +2,11 @@ package mailsac import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Mailsac-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mandrill/mandrill.go b/pkg/detectors/mandrill/mandrill.go index 9b4a16f2efa2..3d25cb04ad85 100644 --- a/pkg/detectors/mandrill/mandrill.go +++ b/pkg/detectors/mandrill/mandrill.go @@ -3,10 +3,11 @@ package mandrill import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/manifest/manifest.go b/pkg/detectors/manifest/manifest.go index f33a6a669ae6..2a4ba92bcda1 100644 --- a/pkg/detectors/manifest/manifest.go +++ b/pkg/detectors/manifest/manifest.go @@ -2,10 +2,11 @@ package manifest import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mapbox/mapbox.go b/pkg/detectors/mapbox/mapbox.go index d2feda74b515..53482d2a387f 100644 --- a/pkg/detectors/mapbox/mapbox.go +++ b/pkg/detectors/mapbox/mapbox.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mapquest/mapquest.go b/pkg/detectors/mapquest/mapquest.go index 30efb0f55e63..e771f2a5338d 100644 --- a/pkg/detectors/mapquest/mapquest.go +++ b/pkg/detectors/mapquest/mapquest.go @@ -2,10 +2,11 @@ package mapquest import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/marketstack/marketstack.go b/pkg/detectors/marketstack/marketstack.go index 1f5c3fc23a96..4ab8bfe421eb 100644 --- a/pkg/detectors/marketstack/marketstack.go +++ b/pkg/detectors/marketstack/marketstack.go @@ -3,10 +3,11 @@ package marketstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go index 28b097bf49b1..73938cc872b0 100644 --- a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go +++ b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken.go @@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mavenlink/mavenlink.go b/pkg/detectors/mavenlink/mavenlink.go index 961ed4c07f19..2474385e6785 100644 --- a/pkg/detectors/mavenlink/mavenlink.go +++ b/pkg/detectors/mavenlink/mavenlink.go @@ -3,10 +3,11 @@ package mavenlink import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/maxmindlicense/v1/maxmindlicense.go b/pkg/detectors/maxmindlicense/v1/maxmindlicense.go index da21ec8998ef..e278a644625c 100644 --- a/pkg/detectors/maxmindlicense/v1/maxmindlicense.go +++ b/pkg/detectors/maxmindlicense/v1/maxmindlicense.go @@ -67,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(idRes, keyRes) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/meaningcloud/meaningcloud.go b/pkg/detectors/meaningcloud/meaningcloud.go index 8428e6877f2d..d912813701fa 100644 --- a/pkg/detectors/meaningcloud/meaningcloud.go +++ b/pkg/detectors/meaningcloud/meaningcloud.go @@ -4,12 +4,13 @@ import ( "bytes" "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "io" "mime/multipart" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -71,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - writer.Close() + _ = writer.Close() req, err := http.NewRequestWithContext(ctx, "POST", "https://api.meaningcloud.com/lang-4.0/identification", bytes.NewReader(body.Bytes())) if err != nil { continue @@ -79,7 +80,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", writer.FormDataContentType()) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { var r response if err := json.NewDecoder(res.Body).Decode(&r); err != nil { diff --git a/pkg/detectors/mediastack/mediastack.go b/pkg/detectors/mediastack/mediastack.go index ac44fe833929..ffacab9adc70 100644 --- a/pkg/detectors/mediastack/mediastack.go +++ b/pkg/detectors/mediastack/mediastack.go @@ -3,11 +3,12 @@ package mediastack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/meistertask/meistertask.go b/pkg/detectors/meistertask/meistertask.go index e6b5e346e121..a8161c67fee1 100644 --- a/pkg/detectors/meistertask/meistertask.go +++ b/pkg/detectors/meistertask/meistertask.go @@ -3,10 +3,11 @@ package meistertask import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mesibo/mesibo.go b/pkg/detectors/mesibo/mesibo.go index 2dcc70344ba2..83e30a89588b 100644 --- a/pkg/detectors/mesibo/mesibo.go +++ b/pkg/detectors/mesibo/mesibo.go @@ -96,7 +96,7 @@ func (s Scanner) verify(ctx context.Context, token string) (bool, error) { if err != nil { return false, fmt.Errorf("failed to execute request: %w", err) } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // The backend API always returns HTTP 200, with the actual result encoded in the // JSON response body. diff --git a/pkg/detectors/messagebird/messagebird.go b/pkg/detectors/messagebird/messagebird.go index 59c164f214f7..6a9e07cbf62a 100644 --- a/pkg/detectors/messagebird/messagebird.go +++ b/pkg/detectors/messagebird/messagebird.go @@ -3,10 +3,11 @@ package messagebird import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("AccessKey %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/metaapi/metaapi.go b/pkg/detectors/metaapi/metaapi.go index 93b8939f7159..301cac6e4997 100644 --- a/pkg/detectors/metaapi/metaapi.go +++ b/pkg/detectors/metaapi/metaapi.go @@ -3,11 +3,12 @@ package metaapi import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -62,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("apikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() body, errBody := io.ReadAll(res.Body) if errBody == nil { diff --git a/pkg/detectors/metabase/metabase.go b/pkg/detectors/metabase/metabase.go index 406bebc7a191..990eb83b5901 100644 --- a/pkg/detectors/metabase/metabase.go +++ b/pkg/detectors/metabase/metabase.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "net/http" + "net/url" "strings" regexp "github.com/wasilibs/go-re2" @@ -65,23 +66,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - u.Path = "/api/user/current" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - continue - } - req.Header.Add("X-Metabase-Session", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - continue - } - if res.StatusCode == http.StatusOK && json.Valid(body) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, u, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -91,6 +78,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, u *url.URL, resMatch string) (bool, error) { + u.Path = "/api/user/current" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return false, err + } + req.Header.Add("X-Metabase-Session", resMatch) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + if res.StatusCode == http.StatusOK && json.Valid(body) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Metabase } diff --git a/pkg/detectors/metrilo/metrilo.go b/pkg/detectors/metrilo/metrilo.go index daa2202c9214..b9d6d96bbdec 100644 --- a/pkg/detectors/metrilo/metrilo.go +++ b/pkg/detectors/metrilo/metrilo.go @@ -2,10 +2,11 @@ package metrilo import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Length", `0`) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook.go similarity index 94% rename from pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go rename to pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook.go index 4a5c993c313e..c1b62a949b96 100644 --- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go +++ b/pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook.go @@ -20,6 +20,9 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (s Scanner) Version() int { return 1 } var ( defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses @@ -50,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/microsoftteams/", + "version": fmt.Sprintf("%d", s.Version()), } if verify { @@ -81,7 +85,7 @@ func verifyWebhook(ctx context.Context, client *http.Client, webhookURL string) if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() body, err := io.ReadAll(res.Body) if err != nil { diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_integration_test.go b/pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook_integration_test.go similarity index 100% rename from pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_integration_test.go rename to pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook_integration_test.go diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go b/pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook_test.go similarity index 100% rename from pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go rename to pkg/detectors/microsoftteamswebhook/v1/microsoftteamswebhook_test.go diff --git a/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook.go new file mode 100644 index 000000000000..679c7841ceef --- /dev/null +++ b/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook.go @@ -0,0 +1,124 @@ +package microsoftteamswebhook + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) +var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) + +func (s Scanner) Version() int { return 2 } + +var ( + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses + + // urlPat matches the base Power Automate webhook URL plus its query string. + // The path is matched strictly; the query string is matched loosely so that + // parameter ordering changes in the future do not break detection. + // Example: https://default..environment.api.powerplatform.com:443/powerautomate/automations/direct/workflows//triggers/manual/paths/invoke?api-version=1&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig= + urlPat = regexp.MustCompile(`https://[a-z0-9]+\.\d+\.environment\.api\.powerplatform\.com(?::\d+)?/powerautomate/automations/direct/workflows/[a-f0-9]{32}/triggers/manual/paths/invoke\?[^\s"'<>]+`) + + // sigPat extracts the sig parameter value from anywhere in the query string. + sigPat = regexp.MustCompile(`[?&]sig=([A-Za-z0-9_\-]+)`) +) + +func (s Scanner) Keywords() []string { + return []string{"environment.api.powerplatform.com"} +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueMatches := make(map[string]struct{}) + for _, urlMatch := range urlPat.FindAllString(dataStr, -1) { + // sig is the signing key that authenticates the request; without it the URL is not a valid credential. + if sigPat.MatchString(urlMatch) { + uniqueMatches[strings.TrimSpace(urlMatch)] = struct{}{} + } + } + + for secret := range uniqueMatches { + r := detectors.Result{ + DetectorType: detector_typepb.DetectorType_MicrosoftTeamsWebhook, + Raw: []byte(secret), + SecretParts: map[string]string{"key": secret}, + ExtraData: map[string]string{ + "version": fmt.Sprintf("%d", s.Version()), + }, + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + isVerified, verificationErr := verifyWebhook(ctx, client, secret) + r.Verified = isVerified + r.SetVerificationError(verificationErr, secret) + } + + results = append(results, r) + } + return results, nil +} + +// verifyWebhook sends a POST request to the webhook URL to verify it is active. +// A 202 response indicates the credential is valid; 400 means the webhook is disabled +// or deleted; 401 means unauthorized. +// The payload sends an empty text and intentionally omits the "type" field so the Power Automate flow accepts +// the request (returning 202) but does not deliver any message to the Teams channel. +func verifyWebhook(ctx context.Context, client *http.Client, webhookURL string) (bool, error) { + payload := strings.NewReader(`{"text":""}`) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, payload) + if err != nil { + return false, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("failed to make request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusAccepted: + return true, nil + case http.StatusUnauthorized, http.StatusBadRequest: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +// IsFalsePositive implements detectors.CustomFalsePositiveChecker. +// The raw value is a full webhook URL, not a short token, so wordlist-based +// false positive detection is not applicable. +func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) { + return false, "" +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_MicrosoftTeamsWebhook +} + +func (s Scanner) Description() string { + return "Microsoft Teams Webhooks (Power Automate) allow external services to communicate with Teams channels by sending messages to a unique Power Automate workflow URL." +} diff --git a/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook_integration_test.go b/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook_integration_test.go new file mode 100644 index 000000000000..dc92a080af42 --- /dev/null +++ b/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package microsoftteamswebhook + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestScanner_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("MICROSOFT_TEAMS_WEBHOOK_V2") + inactiveSecret := testSecrets.MustGetField("MICROSOFT_TEAMS_WEBHOOK_V2_INACTIVE") + + tests := []struct { + name string + s Scanner + data []byte + verify bool + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + data: []byte(fmt.Sprintf("teams webhook %s", secret)), + verify: true, + want: []detectors.Result{ + {DetectorType: detector_typepb.DetectorType_MicrosoftTeamsWebhook, Verified: true}, + }, + }, + { + name: "found, unverified (inactive secret)", + s: Scanner{}, + data: []byte(fmt.Sprintf("teams webhook %s", inactiveSecret)), + verify: true, + want: []detectors.Result{ + {DetectorType: detector_typepb.DetectorType_MicrosoftTeamsWebhook, Verified: false}, + }, + }, + { + name: "found, verification error (unexpected response)", + s: Scanner{client: common.ConstantResponseHttpClient(500, "")}, + data: []byte(fmt.Sprintf("teams webhook %s", secret)), + verify: true, + want: []detectors.Result{{DetectorType: detector_typepb.DetectorType_MicrosoftTeamsWebhook, Verified: false}}, + wantVerificationErr: true, + }, + { + name: "found, verification error (timeout)", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + data: []byte(fmt.Sprintf("teams webhook %s", secret)), + verify: true, + want: []detectors.Result{{DetectorType: detector_typepb.DetectorType_MicrosoftTeamsWebhook, Verified: false}}, + wantVerificationErr: true, + }, + { + name: "not found", + s: Scanner{}, + data: []byte("no secret here"), + verify: true, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(context.Background(), tt.verify, tt.data) + if (err != nil) != tt.wantErr { + t.Fatalf("FromData() error = %v, wantErr %v", err, tt.wantErr) + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: %+v", got[i]) + } + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Errorf("verificationError = %v, wantVerificationErr %v", + got[i].VerificationError(), tt.wantVerificationErr) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "ExtraData", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") + if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { + t.Errorf("FromData() diff (-want +got):\n%s", diff) + } + }) + } +} + +func BenchmarkFromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook_test.go b/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook_test.go new file mode 100644 index 000000000000..6ccb5d506ba5 --- /dev/null +++ b/pkg/detectors/microsoftteamswebhook/v2/microsoftteamswebhook_test.go @@ -0,0 +1,79 @@ +package microsoftteamswebhook + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "https://defaultabc123def456abc123def456ab.62.environment.api.powerplatform.com:443/powerautomate/automations/direct/workflows/67b9621a4a744d4abc90035cb396b361/triggers/manual/paths/invoke?api-version=1&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=r2h9kxq06-gWOJ7QiEHNTxntTw11k2uJA3EZr0SIcIQ" + validPatternReordered = "https://defaultabc123def456abc123def456ab.62.environment.api.powerplatform.com:443/powerautomate/automations/direct/workflows/67b9621a4a744d4abc90035cb396b361/triggers/manual/paths/invoke?sig=r2h9kxq06-gWOJ7QiEHNTxntTw11k2uJA3EZr0SIcIQ&api-version=1&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0" + invalidPattern = "https://defaultabc123.webhook.office.com/webhookb2/not-a-v2-url" + noSigPattern = "https://defaultabc123def456abc123def456ab.62.environment.api.powerplatform.com:443/powerautomate/automations/direct/workflows/67b9621a4a744d4abc90035cb396b361/triggers/manual/paths/invoke?api-version=1&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0" +) + +func TestScanner_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("teams webhook url = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern - sig first", + input: fmt.Sprintf("teams webhook url = '%s'", validPatternReordered), + want: []string{validPatternReordered}, + }, + { + name: "invalid pattern - wrong domain", + input: fmt.Sprintf("webhook = '%s'", invalidPattern), + want: []string{}, + }, + { + name: "invalid pattern - missing sig", + input: fmt.Sprintf("webhook = '%s'", noSigPattern), + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matched := ahoCorasickCore.FindDetectorMatches([]byte(tt.input)) + if len(tt.want) > 0 && len(matched) == 0 { + t.Errorf("keywords not matched") + return + } + results, err := d.FromData(context.Background(), false, []byte(tt.input)) + if err != nil { + t.Fatal(err) + } + actual := make(map[string]struct{}) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}) + for _, v := range tt.want { + expected[v] = struct{}{} + } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("(-want +got)\n%s", diff) + } + }) + } +} diff --git a/pkg/detectors/mindmeister/mindmeister.go b/pkg/detectors/mindmeister/mindmeister.go index 5f9f045dd26b..c8ee0c18638f 100644 --- a/pkg/detectors/mindmeister/mindmeister.go +++ b/pkg/detectors/mindmeister/mindmeister.go @@ -3,11 +3,12 @@ package mindmeister import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"stat":"ok"`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/miro/miro.go b/pkg/detectors/miro/miro.go index 535ea078436a..618d23d30920 100644 --- a/pkg/detectors/miro/miro.go +++ b/pkg/detectors/miro/miro.go @@ -3,10 +3,11 @@ package miro import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mite/mite.go b/pkg/detectors/mite/mite.go index e21554ca8495..c91c75d2ecb4 100644 --- a/pkg/detectors/mite/mite.go +++ b/pkg/detectors/mite/mite.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-MiteApiKey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mixmax/mixmax.go b/pkg/detectors/mixmax/mixmax.go index 0322c59c5cf6..1ab50d09c417 100644 --- a/pkg/detectors/mixmax/mixmax.go +++ b/pkg/detectors/mixmax/mixmax.go @@ -2,10 +2,11 @@ package mixmax import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-Token", resMatch) res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Mixmax, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mixpanel/mixpanel.go b/pkg/detectors/mixpanel/mixpanel.go index 9a2e63d1e59f..76d01a80fd66 100644 --- a/pkg/detectors/mixpanel/mixpanel.go +++ b/pkg/detectors/mixpanel/mixpanel.go @@ -2,10 +2,11 @@ package mixpanel import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mockaroo/mockaroo.go b/pkg/detectors/mockaroo/mockaroo.go index c005331aa435..55e152b453f2 100644 --- a/pkg/detectors/mockaroo/mockaroo.go +++ b/pkg/detectors/mockaroo/mockaroo.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { var t typeRes err = json.NewDecoder(res.Body).Decode(&t) diff --git a/pkg/detectors/moderation/moderation.go b/pkg/detectors/moderation/moderation.go index ade9f0b15e3e..2a12d69d804f 100644 --- a/pkg/detectors/moderation/moderation.go +++ b/pkg/detectors/moderation/moderation.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/monday/monday.go b/pkg/detectors/monday/monday.go index 37e354f9717c..25e698d29a19 100644 --- a/pkg/detectors/monday/monday.go +++ b/pkg/detectors/monday/monday.go @@ -44,18 +44,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Monday, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { isVerified, verificationErr := verifyMondayPAT(ctx, client, resMatch) s1.Verified = isVerified s1.SetVerificationError(verificationErr) - - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": resMatch, - } - } } results = append(results, s1) diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 604d8bdb8f99..b94293ec9020 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -27,7 +27,7 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( - defaultTimeout = 5 * time.Second + defaultTimeout = 10 * time.Second // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. connStrPat = regexp.MustCompile(`\b(mongodb(?:\+srv)?://(?P\S{3,50}):(?P\S{3,88})@(?P[-.%\w]+(?::\d{1,5})?(?:,[-.%\w]+(?::\d{1,5})?)*)(?:/(?P[\w-]+)?(?P\?\w+=[\w@/.$-]+(?:&(?:amp;)?\w+=[\w@/.$-]+)*)?)?)(?:\b|$)`) // TODO: Add support for sharded cluster, replica set and Atlas Deployment. @@ -45,7 +45,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result logger := logContext.AddLogger(ctx).Logger().WithName("mongodb") dataStr := string(data) - uniqueMatches := make(map[string]string) + type mongoMatch struct { + password string + parsedURL *url.URL + } + uniqueMatches := make(map[string]mongoMatch) for _, match := range connStrPat.FindAllStringSubmatch(dataStr, -1) { // Filter out common placeholder passwords. password := match[3] @@ -54,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } // If the query string contains `&` the options will not be parsed. - connStr := strings.Replace(strings.TrimSpace(match[1]), "&", "&", -1) + connStr := strings.ReplaceAll(strings.TrimSpace(match[1]), "&", "&") connUrl, err := url.Parse(connStr) if err != nil { logger.V(3).Info("Skipping invalid URL", "err", err) @@ -78,16 +82,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result connUrl.RawQuery = params.Encode() connStr = connUrl.String() - uniqueMatches[connStr] = password + uniqueMatches[connStr] = mongoMatch{password: password, parsedURL: connUrl} } - for connStr, password := range uniqueMatches { + for connStr, m := range uniqueMatches { + extraData := map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", + } + if m.parsedURL.Host != "" { + extraData["host"] = m.parsedURL.Host + } + if m.parsedURL.User != nil && m.parsedURL.User.Username() != "" { + extraData["username"] = m.parsedURL.User.Username() + } + if db := strings.TrimPrefix(m.parsedURL.Path, "/"); db != "" { + extraData["database"] = db + } + r := detectors.Result{ DetectorType: detector_typepb.DetectorType_MongoDB, Raw: []byte(connStr), - ExtraData: map[string]string{ - "rotation_guide": "https://howtorotate.com/docs/tutorials/mongo/", - }, + ExtraData: extraData, + SecretParts: map[string]string{"key": connStr}, } if verify { @@ -98,15 +114,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, vErr := verifyUri(ctx, connStr, timeout) r.Verified = isVerified - if isErrDeterminate(vErr) { - continue - } - r.SetVerificationError(vErr, password) - - if isVerified { - r.SecretParts = map[string]string{ - "key": connStr, - } + // An auth error means the server tested the credential and rejected it. That is a determinate result, + // so it is reported as unverified with no verification error attached. Anything else, + // a timeout, an unreachable host, a malformed connection string is indeterminate and keeps its + // error so the credential is retried on a later scan. + if !isErrDeterminate(vErr) { + r.SetVerificationError(vErr, m.password) } } results = append(results, r) diff --git a/pkg/detectors/mongodb/mongodb_integration_test.go b/pkg/detectors/mongodb/mongodb_integration_test.go index 60a6b5e658bf..fa1557f5cd36 100644 --- a/pkg/detectors/mongodb/mongodb_integration_test.go +++ b/pkg/detectors/mongodb/mongodb_integration_test.go @@ -173,7 +173,7 @@ func TestIntegrationMongoDB_FromChunk(t *testing.T) { t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } @@ -310,7 +310,7 @@ func TestMongoDB_FromChunk(t *testing.T) { t.Fatalf("wantVerificationErr = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { t.Errorf("MongoDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.go index 9482fcca7cf3..2836bcff0ef8 100644 --- a/pkg/detectors/mongodb/mongodb_test.go +++ b/pkg/detectors/mongodb/mongodb_test.go @@ -3,8 +3,69 @@ package mongodb import ( "context" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestMongoDB_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "single host with port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017`, + wantHost: "mongodb0.example.com:27017", + wantUsername: "myDBReader", + }, + { + name: "single host without port", + data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com`, + wantHost: "mongodb0.example.com", + wantUsername: "myDBReader", + }, + { + name: "with options and no database", + data: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true`, + wantHost: "host.docker.internal:27018", + wantUsername: "username", + }, + { + name: "cosmos db style with database", + data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/csb-db?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`, + wantHost: "agenda-live.mongo.cosmos.azure.com:10255", + wantUsername: "agenda-live", + wantDatabase: "csb-db", + }, + { + name: "with database in path", + data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`, + wantHost: "mongodb-instance:27017", + wantUsername: "db-user", + wantDatabase: "db-name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + assert.NotEmpty(t, r.ExtraData["rotation_guide"], "ExtraData[rotation_guide] should still be present") + }) + } +} + func TestMongoDB_Pattern(t *testing.T) { tests := []struct { name string diff --git a/pkg/detectors/monkeylearn/monkeylearn.go b/pkg/detectors/monkeylearn/monkeylearn.go index 938be4f77e72..a2a1ed2e2f7f 100644 --- a/pkg/detectors/monkeylearn/monkeylearn.go +++ b/pkg/detectors/monkeylearn/monkeylearn.go @@ -3,10 +3,11 @@ package monkeylearn import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/moonclerk/moonclerk.go b/pkg/detectors/moonclerk/moonclerk.go index 3bf6115dc39b..1014d6c34880 100644 --- a/pkg/detectors/moonclerk/moonclerk.go +++ b/pkg/detectors/moonclerk/moonclerk.go @@ -2,10 +2,11 @@ package moonclerk import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", "Token token="+resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/moosend/moosend.go b/pkg/detectors/moosend/moosend.go index 75e5b8446905..489b3d7514de 100644 --- a/pkg/detectors/moosend/moosend.go +++ b/pkg/detectors/moosend/moosend.go @@ -2,11 +2,12 @@ package moosend import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,22 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.moosend.com/v3/lists.json?apikey="+resMatch+"&format=json", nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if strings.Contains(body, "CreatedOn") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -70,6 +58,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.moosend.com/v3/lists.json?apikey="+resMatch+"&format=json", nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if strings.Contains(body, "CreatedOn") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Moosend } diff --git a/pkg/detectors/moralis/moralis.go b/pkg/detectors/moralis/moralis.go index ae505af22dc0..eb74e6e0f201 100644 --- a/pkg/detectors/moralis/moralis.go +++ b/pkg/detectors/moralis/moralis.go @@ -2,10 +2,11 @@ package moralis import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock.go index 4452a5713c8f..d4d497243024 100644 --- a/pkg/detectors/mrticktock/mrticktock.go +++ b/pkg/detectors/mrticktock/mrticktock.go @@ -57,24 +57,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := strings.NewReader(fmt.Sprintf(`email=%s&password=%s`, emailMatch, resPassword)) - req, err := http.NewRequestWithContext(ctx, "POST", "https://mrticktock.com/app/api/is_timer_active", payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"errors":[]`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, emailMatch, resPassword) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, emailMatch, resPassword) } results = append(results, s1) @@ -83,6 +68,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, emailMatch string, resPassword string) (bool, error) { + payload := strings.NewReader(fmt.Sprintf(`email=%s&password=%s`, emailMatch, resPassword)) + req, err := http.NewRequestWithContext(ctx, "POST", "https://mrticktock.com/app/api/is_timer_active", payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"errors":[]`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Mrticktock } diff --git a/pkg/detectors/mux/mux.go b/pkg/detectors/mux/mux.go index 3f0781fcc718..d04a949bd832 100644 --- a/pkg/detectors/mux/mux.go +++ b/pkg/detectors/mux/mux.go @@ -50,6 +50,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_Mux, Raw: []byte(resMatch), RawV2: []byte(resMatch + resSecretMatch), + SecretParts: map[string]string{ + "key": resMatch, + "secret": resSecretMatch, + }, } if verify { @@ -61,17 +65,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, resSecretMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } } - if s1.Verified { - s1.SecretParts = map[string]string{ - "key": resMatch, - "secret": resSecretMatch, - } - } } results = append(results, s1) diff --git a/pkg/detectors/myfreshworks/myfreshworks.go b/pkg/detectors/myfreshworks/myfreshworks.go index 9790fb78c753..307dadbec047 100644 --- a/pkg/detectors/myfreshworks/myfreshworks.go +++ b/pkg/detectors/myfreshworks/myfreshworks.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -86,7 +87,7 @@ func verifyMyfreshworks(ctx context.Context, client *http.Client, resMatch, resI if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode == http.StatusOK { body, err := io.ReadAll(res.Body) @@ -95,7 +96,7 @@ func verifyMyfreshworks(ctx context.Context, client *http.Client, resMatch, resI } return json.Valid(body), nil - } else if !(res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden) { + } else if res.StatusCode != http.StatusUnauthorized && res.StatusCode != http.StatusForbidden { return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } diff --git a/pkg/detectors/myintervals/myintervals.go b/pkg/detectors/myintervals/myintervals.go index d802115b4051..5887442a890f 100644 --- a/pkg/detectors/myintervals/myintervals.go +++ b/pkg/detectors/myintervals/myintervals.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nasdaqdatalink/nasdaqdatalink.go b/pkg/detectors/nasdaqdatalink/nasdaqdatalink.go index 283b8a1f53be..f4ea104a4aec 100644 --- a/pkg/detectors/nasdaqdatalink/nasdaqdatalink.go +++ b/pkg/detectors/nasdaqdatalink/nasdaqdatalink.go @@ -3,11 +3,12 @@ package nasdaqdatalink import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nethunt/nethunt.go b/pkg/detectors/nethunt/nethunt.go index 649e613ddd44..d68978b26829 100644 --- a/pkg/detectors/nethunt/nethunt.go +++ b/pkg/detectors/nethunt/nethunt.go @@ -2,10 +2,11 @@ package nethunt import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/netlify/v1/netlify_v1.go b/pkg/detectors/netlify/v1/netlify_v1.go index aeedae839aba..ffd72c5b8948 100644 --- a/pkg/detectors/netlify/v1/netlify_v1.go +++ b/pkg/detectors/netlify/v1/netlify_v1.go @@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Netlify, Raw: []byte(match), + SecretParts: map[string]string{"key": match}, } s1.ExtraData = map[string]string{ "rotation_guide": rotationGuideUrl, @@ -62,10 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyMatch(ctx, client, match) s1.Verified = isVerified s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{"key": match} - } } results = append(results, s1) diff --git a/pkg/detectors/netlify/v2/netlify_v2.go b/pkg/detectors/netlify/v2/netlify_v2.go index 63edf985e263..8333a2bab9a8 100644 --- a/pkg/detectors/netlify/v2/netlify_v2.go +++ b/pkg/detectors/netlify/v2/netlify_v2.go @@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Netlify, Raw: []byte(match), + SecretParts: map[string]string{"key": match}, } s1.ExtraData = map[string]string{ "rotation_guide": rotationGuideUrl, @@ -62,10 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyMatch(ctx, client, match) s1.Verified = isVerified s1.SetVerificationError(verificationErr, match) - - if s1.Verified { - s1.SecretParts = map[string]string{"key": match} - } } results = append(results, s1) diff --git a/pkg/detectors/netsuite/netsuite.go b/pkg/detectors/netsuite/netsuite.go index 10f44889599c..41b8a04b679a 100644 --- a/pkg/detectors/netsuite/netsuite.go +++ b/pkg/detectors/netsuite/netsuite.go @@ -7,7 +7,10 @@ import ( "crypto/sha256" "encoding/base64" "encoding/binary" + "errors" "fmt" + "io" + "net" "net/http" "net/url" "strconv" @@ -16,6 +19,7 @@ import ( regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -41,6 +45,9 @@ var ( tokenSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netsuite", "token", "secret"}) + `\b([a-zA-Z0-9]{64})\b`) accountIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netsuite", "account", "id"}) + `\b([a-zA-Z0-9-_]{6,15})\b`) + + // invalidHosts holds account hosts that do not resolve, so later combinations skip the lookup. + invalidHosts = simple.NewCache[struct{}]() ) type credentialSet struct { @@ -68,8 +75,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result tokenSecretMatches := trimUniqueMatches(tokenSecretPat.FindAllStringSubmatch(dataStr, -1)) accountIDMatches := trimUniqueMatches(accountIDPat.FindAllStringSubmatch(dataStr, -1)) + // Every combination needs a token key, token secret and account ID. + if len(tokenKeyMatches) == 0 || len(tokenSecretMatches) == 0 || len(accountIDMatches) == 0 { + return nil, nil + } + + // Raw and RawV2 hold only the consumer key and secret, so every combination for a pair is a duplicate of the same + // finding. Report one result per pair rather than one per combination of all five parts. The token key, token + // secret and account ID decide whether the pair is reported and whether it verifies. for consumerKey := range consumerKeyMatches { for consumerSecret := range consumerSecretMatches { + // No combination can use the same value as both consumer key and secret. + if consumerKey == consumerSecret { + continue + } + + var pairResult *detectors.Result + combinations: for tokenKey := range tokenKeyMatches { for tokenSecret := range tokenSecretMatches { for accountID := range accountIDMatches { @@ -107,10 +129,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.SetVerificationError(err, consumerKey) } - results = append(results, s1) + + // Keep the verified result, else the first indeterminate one, else the first one. + if pairResult == nil || s1.Verified || (s1.VerificationError() != nil && pairResult.VerificationError() == nil) { + pairResult = &s1 + } + if !verify || s1.Verified { + break combinations + } + if err := ctx.Err(); err != nil { + // Combinations left untried might have verified. + if pairResult.VerificationError() == nil { + pairResult.SetVerificationError(err, consumerKey) + } + break combinations + } } } } + + if pairResult != nil { + results = append(results, *pairResult) + } } } @@ -126,8 +166,19 @@ func (s Scanner) Description() string { } func verifyCredentials(ctx context.Context, client *http.Client, cs credentialSet) (bool, error) { + // The engine waits for FromData to return even after its deadline, so send no request once the context is done. + if err := ctx.Err(); err != nil { + return false, err + } + // for url, filter or replace underscore in accountID if needed and lower case the accountID - urlAccountId := strings.ToLower(strings.Replace(cs.accountID, "_", "-", -1)) + urlAccountId := strings.ToLower(strings.ReplaceAll(cs.accountID, "_", "-")) + + // An earlier combination already found that this account's host does not resolve, so skip the lookup and + // return what the lookup would. + if invalidHosts.Exists(urlAccountId) { + return false, nil + } baseUrl := "https://" + urlAccountId + ".suitetalk.api.netsuite.com" @@ -164,12 +215,17 @@ func verifyCredentials(ctx context.Context, client *http.Client, cs credentialSe // Make the request res, err := client.Do(req) if err != nil { - if strings.Contains(err.Error(), "no such host") { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && dnsErr.IsNotFound { + invalidHosts.Set(urlAccountId, struct{}{}) return false, nil } return false, err } - defer res.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() switch res.StatusCode { case http.StatusOK: return true, nil diff --git a/pkg/detectors/netsuite/netsuite_integration_test.go b/pkg/detectors/netsuite/netsuite_integration_test.go index 85d2495d9e2e..82d5f5ceb410 100644 --- a/pkg/detectors/netsuite/netsuite_integration_test.go +++ b/pkg/detectors/netsuite/netsuite_integration_test.go @@ -78,7 +78,7 @@ func TestNetsuite_FromChunk(t *testing.T) { verify: true, }, ShouldHaveVerified: false, - wantCount: 21, + wantCount: 5, wantErr: false, }, { diff --git a/pkg/detectors/netsuite/netsuite_test.go b/pkg/detectors/netsuite/netsuite_test.go index 62e44d4f0425..237dd76fa3c9 100644 --- a/pkg/detectors/netsuite/netsuite_test.go +++ b/pkg/detectors/netsuite/netsuite_test.go @@ -3,10 +3,16 @@ package netsuite import ( "context" "fmt" + "io" + "net" + "net/http" + "strings" + "sync/atomic" "testing" "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -41,7 +47,7 @@ func TestNetsuite_Pattern(t *testing.T) { { name: "valid pattern - with keyword netsuite", input: fmt.Sprintf(inputFormat, keyword, validAccountID, validConsumerKey, validConsumerSecret, validTokenKey, validTokenSecret), - want: []string{outputPair1, outputPair2, outputPair1, outputPair2}, + want: []string{outputPair1, outputPair2}, }, { name: "invalid pattern", @@ -92,3 +98,202 @@ func TestNetsuite_Pattern(t *testing.T) { }) } } + +// denseValue returns the i-th 64-character value of tokenDenseInput. +func denseValue(i int) string { + return fmt.Sprintf("%064x", i) +} + +// tokenDenseInput returns n distinct 64-character values, each of which every key and secret pattern matches, and one +// account ID. +func tokenDenseInput(n int) []byte { + var input strings.Builder + for i := 1; i <= n; i++ { + fmt.Fprintf(&input, "netsuite = %s\n", denseValue(i)) + } + input.WriteString("account_id = 1234567\n") + return []byte(input.String()) +} + +// fakeClient returns a client that counts its requests and answers each with the status respond picks. +func fakeClient(requests *atomic.Int32, respond func(req *http.Request) int) *http.Client { + return &http.Client{ + Transport: common.FakeTransport{ + CreateResponse: func(req *http.Request) (*http.Response, error) { + requests.Add(1) + return &http.Response{ + Request: req, + StatusCode: respond(req), + Body: io.NopCloser(strings.NewReader("")), + }, nil + }, + }, + } +} + +func TestNetsuite_TokenDenseInput(t *testing.T) { + const values = 16 + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + results, err := Scanner{}.FromData(ctx, false, tokenDenseInput(values)) + if err != nil { + t.Fatalf("error = %v", err) + } + + // One result per ordered consumer key and secret pair, not one per combination of all five parts. + if want := values * (values - 1); len(results) != want { + t.Fatalf("expected %d results, got %d", want, len(results)) + } + seen := make(map[string]struct{}, len(results)) + for _, r := range results { + if _, ok := seen[string(r.RawV2)]; ok { + t.Fatalf("duplicate result %q", r.RawV2) + } + seen[string(r.RawV2)] = struct{}{} + } +} + +func TestNetsuite_Verification(t *testing.T) { + // With 4 values there are 4*3 consumer key and secret pairs, each completed by 2*1 token key and secret orders. + tests := []struct { + name string + values int + status int + cancel bool + wantVerified bool + wantErr string + wantRequests int32 + }{ + { + name: "rejected pairs try every combination", + values: 4, + status: http.StatusUnauthorized, + wantRequests: 12 * 2, + }, + { + name: "verification stops at the first accepted combination", + values: 4, + status: http.StatusOK, + wantVerified: true, + wantRequests: 12, + }, + { + name: "canceled context sends no requests", + values: 16, + status: http.StatusOK, + cancel: true, + wantErr: context.Canceled.Error(), + wantRequests: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var requests atomic.Int32 + client := fakeClient(&requests, func(*http.Request) int { return tt.status }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tt.cancel { + cancel() + } + + results, err := Scanner{client: client}.FromData(ctx, true, tokenDenseInput(tt.values)) + if err != nil { + t.Fatalf("error = %v", err) + } + + if want := tt.values * (tt.values - 1); len(results) != want { + t.Fatalf("expected %d results, got %d", want, len(results)) + } + for _, r := range results { + if r.Verified != tt.wantVerified { + t.Fatalf("verified = %v, want %v", r.Verified, tt.wantVerified) + } + var gotErr string + if err := r.VerificationError(); err != nil { + gotErr = err.Error() + } + if gotErr != tt.wantErr { + t.Fatalf("verification error = %q, want %q", gotErr, tt.wantErr) + } + } + if got := requests.Load(); got != tt.wantRequests { + t.Errorf("requests = %d, want %d", got, tt.wantRequests) + } + }) + } +} + +func TestNetsuite_VerificationFindsTheAcceptedCombination(t *testing.T) { + // Accept only consumer key 1 with token key 4, so each pair has at most one accepted combination among the + // rejected ones. + var requests atomic.Int32 + client := fakeClient(&requests, func(req *http.Request) int { + auth := req.Header.Get("Authorization") + if strings.Contains(auth, `oauth_consumer_key="`+denseValue(1)+`"`) && + strings.Contains(auth, `oauth_token="`+denseValue(4)+`"`) { + return http.StatusOK + } + return http.StatusUnauthorized + }) + + results, err := Scanner{client: client}.FromData(context.Background(), true, tokenDenseInput(4)) + if err != nil { + t.Fatalf("error = %v", err) + } + + // Token key 4 completes consumer key 1 with secret 2 or 3, but not with secret 4. + want := map[string]struct{}{ + denseValue(1) + denseValue(2): {}, + denseValue(1) + denseValue(3): {}, + } + got := make(map[string]struct{}) + for _, r := range results { + if r.Verified { + got[string(r.RawV2)] = struct{}{} + } + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("verified pairs diff: (-want +got)\n%s", diff) + } +} + +func TestNetsuite_VerificationMissingHost(t *testing.T) { + invalidHosts.Clear() + t.Cleanup(invalidHosts.Clear) + + var requests atomic.Int32 + client := &http.Client{ + Transport: common.FakeTransport{ + CreateResponse: func(req *http.Request) (*http.Response, error) { + requests.Add(1) + return nil, &net.DNSError{Err: "no such host", Name: req.URL.Hostname(), IsNotFound: true} + }, + }, + } + s := Scanner{client: client} + + // Scan twice, as if the data were split across two chunks. A missing host is unverified, as before. + for range 2 { + results, err := s.FromData(context.Background(), true, tokenDenseInput(4)) + if err != nil { + t.Fatalf("error = %v", err) + } + if len(results) != 12 { + t.Fatalf("expected 12 results, got %d", len(results)) + } + for _, r := range results { + if r.Verified || r.VerificationError() != nil { + t.Fatalf("verified = %v, verification error = %v, want unverified", r.Verified, r.VerificationError()) + } + } + } + + // The first lookup caches the host, so no later combination or chunk looks it up again. + if got := requests.Load(); got != 1 { + t.Errorf("requests = %d, want 1", got) + } +} diff --git a/pkg/detectors/neutrinoapi/neutrinoapi.go b/pkg/detectors/neutrinoapi/neutrinoapi.go index 5bcbd0c1cbfb..16a577c0abe3 100644 --- a/pkg/detectors/neutrinoapi/neutrinoapi.go +++ b/pkg/detectors/neutrinoapi/neutrinoapi.go @@ -4,12 +4,13 @@ import ( "bytes" "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "mime/multipart" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -67,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - writer.Close() + _ = writer.Close() req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://neutrinoapi.net/url-info?user-id=%s&api-key=%s", resIdMatch, resMatch), bytes.NewReader(body.Bytes())) if err != nil { continue @@ -75,7 +76,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", writer.FormDataContentType()) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey.go b/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey.go new file mode 100644 index 000000000000..77a1bdefd873 --- /dev/null +++ b/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey.go @@ -0,0 +1,135 @@ +package newrelicbrowserkey + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(`\b(NRBR-[0-9a-f]{19})\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"nrbr-"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicBrowserKey +} + +func (s Scanner) Description() string { + return "A New Relic Browser API key is used to authenticate and enable browser monitoring, allowing New Relic to collect performance data, page views, and user interactions from web applications to track and optimize front-end performance." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + Redacted: resMatch[:8] + "...", + SecretParts: map[string]string{"key": resMatch}, + } + + if verify { + isVerified, extraData, verificationErr := s.verify(ctx, resMatch) + s1.Verified = isVerified + s1.ExtraData = extraData + if _, ok := extraData["region"]; ok { + s1.SecretParts["region"] = extraData["region"] + } + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, nil +} + +// verify checks if the key is valid by making a request to New Relic's Browser API endpoint. +// There are separate endpoints for US and EU region keys, so we check both. +// These endpoints are not documented anywhere because they are used internally by the New Relic Browser agent +// The endpoints were discovered by observing network traffic from the New Relic Browser agent +// A 400 Bad Request response indicates that the key is valid but the request is malformed (because we are not sending the expected payload) +// A 403 Forbidden response indicates that the key is invalid or revoked +func (s Scanner) verify(ctx context.Context, key string) (bool, map[string]string, error) { + regionUrls := map[string]string{ + "us": "https://bam.nr-data.net/1/", + "eu": "https://bam.eu01.nr-data.net/1/", + } + errs := make([]error, 0, len(regionUrls)) + for region, regionUrl := range regionUrls { + verified, err := s.verifyRegion(ctx, key, regionUrl) + if err != nil { + errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err)) + continue + } + if verified { + return true, map[string]string{"region": region}, nil + } + } + return false, nil, errors.Join(errs...) +} + +func (s Scanner) verifyRegion(ctx context.Context, key string, regionUrl string) (bool, error) { + fullUrl, err := url.JoinPath(regionUrl, key) + if err != nil { + return false, fmt.Errorf("error constructing URL: %w", err) + } + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, fullUrl, http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + + client := s.getClient() + + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusBadRequest: + return true, nil + case http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey_integration_test.go b/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey_integration_test.go new file mode 100644 index 000000000000..89d5516c9865 --- /dev/null +++ b/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey_integration_test.go @@ -0,0 +1,151 @@ +//go:build detectors +// +build detectors + +package newrelicbrowserkey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicBrowserKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_BROWSER_KEY") + keyEU := testSecrets.MustGetField("NEW_RELIC_BROWSER_KEY_EU") + keyInactive := "NRBR-cd83c5e6c53fe2edc1a" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic browser key %s within", key)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicBrowserKey, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{"key": key, "region": "us"}, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic browser key %s within", keyEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicBrowserKey, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + SecretParts: map[string]string{"key": keyEU, "region": "eu"}, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic browser key %s within", keyInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicBrowserKey, + Verified: false, + SecretParts: map[string]string{"key": keyInactive}, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicBrowserKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicBrowserKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey_test.go b/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey_test.go new file mode 100644 index 000000000000..7438bd2e1d68 --- /dev/null +++ b/pkg/detectors/newrelicbrowserkey/newrelicbrowserkey_test.go @@ -0,0 +1,80 @@ +package newrelicbrowserkey + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "NRBR-cd83c5e6c53fe2edc1a" + invalidPattern = "NRBR-cd83c5e6c53fe2edc1" +) + +func TestNewRelicBrowserKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic browser key = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic browser key = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey.go b/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey.go new file mode 100644 index 000000000000..e5eabbbe8786 --- /dev/null +++ b/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey.go @@ -0,0 +1,127 @@ +package newrelicinsightsinsertkey + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(`\b(NRII-[a-zA-Z0-9-_]{25})`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"nrii-"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicInsightsInsertKey +} + +func (s Scanner) Description() string { + return "A New Relic Insights Insert Key is an authentication token used to send event data (such as custom events, logs, and metrics) to New Relic Insights for analysis and visualization. It ensures secure data ingestion from your applications and services." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + Redacted: resMatch[:8] + "...", + SecretParts: map[string]string{ + "key": resMatch, + }, + } + + if verify { + isVerified, extraData, verificationErr := s.verify(ctx, resMatch) + s1.Verified = isVerified + s1.ExtraData = extraData + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, nil +} + +// verify checks if the provided key is valid by making a request to the New Relic Insights API. +// It sends a POST request to the events endpoint. A valid key will result in a 200 OK response, while an invalid key will return a 403 Forbidden. +// Even though the response is 200, no data is actually published to New Relic since the request body is empty. +// https://docs.newrelic.com/docs/data-apis/ingest-apis/event-api/introduction-event-api/ +func (s Scanner) verify(ctx context.Context, key string) (bool, map[string]string, error) { + regionUrls := map[string]string{ + "us": "https://insights-collector.newrelic.com/v1/accounts/`nowaythiscanexist/events", + "eu": "https://insights-collector.eu01.nr-data.net/v1/accounts/`nowaythiscanexist/events", + } + errs := make([]error, 0, len(regionUrls)) + for region, regionUrl := range regionUrls { + verified, err := s.verifyRegion(ctx, key, regionUrl) + if err != nil { + errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err)) + continue + } + if verified { + return true, map[string]string{"region": region}, nil + } + } + return false, nil, errors.Join(errs...) +} + +func (s Scanner) verifyRegion(ctx context.Context, key, regionUrl string) (bool, error) { + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, regionUrl, http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Set("X-Insert-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_integration_test.go b/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_integration_test.go new file mode 100644 index 000000000000..fb82e557b18a --- /dev/null +++ b/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_integration_test.go @@ -0,0 +1,152 @@ +//go:build detectors +// +build detectors + +package newrelicinsightsinsertkey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicInsightsInsertKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_INSERT_KEY") + keyEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_INSERT_KEY_EU") + keyInactive := "NRII-d-2Vf-L1w-8B9Y_--6x8-_QjA" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic insights insert key %s within", key)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsInsertKey, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic insights insert key %s within", keyEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsInsertKey, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic insights insert key %s within", keyInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsInsertKey, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicInsightsInsertKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: \n %+v", got[i]) + } + got[i].SecretParts = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicInsightsInsertKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_test.go b/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_test.go new file mode 100644 index 000000000000..c61fa138e3be --- /dev/null +++ b/pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_test.go @@ -0,0 +1,80 @@ +package newrelicinsightsinsertkey + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "NRII-d-2Vf-L1w-8B9Y_--6x8-_QjA" + invalidPattern = "NRII-d-2Vf-L1w-8B9Y_--6x8-_Qj" +) + +func TestNewRelicInsightsInsertKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic insights insert key = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic insights insert key = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey.go b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey.go new file mode 100644 index 000000000000..913e53af6709 --- /dev/null +++ b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey.go @@ -0,0 +1,144 @@ +package newrelicinsightsquerykey + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider + + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(`\b(NRIQ-[a-zA-Z0-9-_]{25})`) + accountIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"relic", "account", "id"}) + `\b(\d{4,10})\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"nriq-"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicInsightsQueryKey +} + +func (s Scanner) Description() string { + return "A New Relic Insights Query Key is a read-only API key used to execute NRQL queries against your account's event data via the legacy Insights Query API. It allows secure retrieval of analytics data without permitting any data ingestion or modification." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1) + accountIDMatches := accountIDPat.FindAllStringSubmatch(dataStr, -1) + uniqueAccountIDMatches := make(map[string]struct{}) + for _, match := range accountIDMatches { + uniqueAccountIDMatches[match[1]] = struct{}{} + } + + for _, keyMatch := range keyMatches { + for accountID := range uniqueAccountIDMatches { + keyResMatch := strings.TrimSpace(keyMatch[1]) + accountIDResMatch := strings.TrimSpace(accountID) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(keyResMatch), + RawV2: []byte(keyResMatch + accountIDResMatch), + Redacted: keyResMatch[:8] + "...", + SecretParts: map[string]string{ + "key": keyResMatch, + "account_id": accountIDResMatch, + }, + } + + if verify { + isVerified, extraData, verificationErr := s.verify(ctx, keyResMatch, accountIDResMatch) + s1.Verified = isVerified + s1.ExtraData = extraData + if extraData != nil { + s1.SecretParts["region"] = extraData["region"] + } + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + } + + return results, nil +} + +// verify checks if the provided key is valid by making a request to the New Relic Insights Query API. +// It checks both the US and EU endpoints before returning an error. +// Account ID is required to verify as the API endpoint is account-specific. +func (s Scanner) verify(ctx context.Context, key string, accountID string) (bool, map[string]string, error) { + regionUrls := map[string]string{ + "us": fmt.Sprintf("https://insights-api.newrelic.com/v1/accounts/%s/query?nrql=SELECT%%201", accountID), + "eu": fmt.Sprintf("https://insights-api.eu.newrelic.com/v1/accounts/%s/query?nrql=SELECT%%201", accountID), + } + errs := make([]error, 0, len(regionUrls)) + for region, regionUrl := range regionUrls { + verified, err := s.verifyRegion(ctx, key, regionUrl) + if err != nil { + errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err)) + continue + } + if verified { + return true, map[string]string{"region": region}, nil + } + } + return false, nil, errors.Join(errs...) +} + +func (s Scanner) verifyRegion(ctx context.Context, key, regionUrl string) (bool, error) { + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, regionUrl, http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Set("X-Query-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_integration_test.go b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_integration_test.go new file mode 100644 index 000000000000..83f8845d12f7 --- /dev/null +++ b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_integration_test.go @@ -0,0 +1,168 @@ +//go:build detectors +// +build detectors + +package newrelicinsightsquerykey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicInsightsQueryKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_QUERY_KEY") + accountID := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_ACCOUNT_ID") + keyEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_QUERY_KEY_EU") + accountIDEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_ACCOUNT_ID_EU") + keyInactive := "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ8_" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic insights query key %s and account ID %s within", key, accountID)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": key, + "account_id": accountID, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic insights query key %s and account ID %s within", keyEU, accountIDEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + SecretParts: map[string]string{ + "key": keyEU, + "account_id": accountIDEU, + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic insights query key %s and account ID %s within", keyInactive, accountID)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey, + Verified: false, + SecretParts: map[string]string{ + "key": keyInactive, + "account_id": accountID, + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicInsightsQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].RawV2) == 0 { + t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + } + got[i].RawV2 = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicInsightsQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_test.go b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_test.go new file mode 100644 index 000000000000..817c70c30182 --- /dev/null +++ b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_test.go @@ -0,0 +1,81 @@ +package newrelicinsightsquerykey + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ7_" + invalidPattern = "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ7" + accountID = "7746934" +) + +func TestNewRelicInsightsQueryKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic insights query key = '%s' account ID = '%s'", validPattern, accountID), + want: []string{validPattern + accountID}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic insights query key = '%s' account ID = '%s'", invalidPattern, accountID), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/newreliclicensekey/newreliclicensekey.go b/pkg/detectors/newreliclicensekey/newreliclicensekey.go new file mode 100644 index 000000000000..9c98bb691227 --- /dev/null +++ b/pkg/detectors/newreliclicensekey/newreliclicensekey.go @@ -0,0 +1,121 @@ +package newreliclicensekey + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + // https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + // US region keys are 40 characters hexadecimal strings ending with "FFFFNRAL" + // EU region keys have the same format but first 6 characters are "eu01xx" + keyPat = regexp.MustCompile(`\b(([0-9a-f]{32}|eu01xx[0-9a-f]{26})FFFFNRAL)\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"ffffnral"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicLicenseKey +} + +func (s Scanner) Description() string { + return "New Relic license keys are unique authentication tokens used to send telemetry data (metrics, logs, and traces) from your applications and infrastructure to New Relic." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + Redacted: resMatch[:8] + "...", + SecretParts: map[string]string{"key": resMatch}, + } + + if strings.HasPrefix(resMatch, "eu01xx") { + s1.SecretParts["region"] = "eu" + s1.ExtraData = map[string]string{"region": "eu"} + } else { + s1.SecretParts["region"] = "us" + s1.ExtraData = map[string]string{"region": "us"} + } + + if verify { + isVerified, verificationErr := s.verify(ctx, resMatch, s1.SecretParts["region"]) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, nil +} + +// verify checks if the provided key is valid by making a request to the New Relic Metrics API. +// It sends a POST request to the metrics endpoint. A valid key will result in a 202 Accepted response, while an invalid key will return a 403 Forbidden. +// Even though the response is 202, no data is actually published to New Relic since the request body is empty. +// https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/ +func (s Scanner) verify(ctx context.Context, key, region string) (bool, error) { + host := "https://metric-api.newrelic.com" + if region == "eu" { + // EU region keys have a different host + host = "https://metric-api.eu.newrelic.com" + } + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, host+"/metric/v1", http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Api-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusAccepted: + return true, nil + case http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newreliclicensekey/newreliclicensekey_integration_test.go b/pkg/detectors/newreliclicensekey/newreliclicensekey_integration_test.go new file mode 100644 index 000000000000..e5ca7a8de33d --- /dev/null +++ b/pkg/detectors/newreliclicensekey/newreliclicensekey_integration_test.go @@ -0,0 +1,162 @@ +//go:build detectors +// +build detectors + +package newreliclicensekey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicLicenseKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_LICENSE_KEY") + keyEU := testSecrets.MustGetField("NEW_RELIC_LICENSE_KEY_EU") + keyInactive := "72322bc2443d330cf29cde9f24fca105FFFFNRAL" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic license key %s within", key)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicLicenseKey, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": key, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic license key %s within", keyEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicLicenseKey, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + SecretParts: map[string]string{ + "key": keyEU, + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic license key %s within", keyInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicLicenseKey, + Verified: false, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": keyInactive, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicLicenseKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicLicenseKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newreliclicensekey/newreliclicensekey_test.go b/pkg/detectors/newreliclicensekey/newreliclicensekey_test.go new file mode 100644 index 000000000000..b62ec78e16b8 --- /dev/null +++ b/pkg/detectors/newreliclicensekey/newreliclicensekey_test.go @@ -0,0 +1,86 @@ +package newreliclicensekey + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "72322bc2443d330cf29cde9f24fca105FFFFNRAL" + validPatternEU = "eu01xxb7e8b0dddc28ac051a64ffd583FFFFNRAL" + invalidPattern = "72322bc2443d330cf29cde9f24fca5FFFFNRAL" +) + +func TestNewRelicLicenseKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic license key = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern EU", + input: fmt.Sprintf("new relic license key EU = '%s'", validPatternEU), + want: []string{validPatternEU}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic license key = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go new file mode 100644 index 000000000000..233e306ffec5 --- /dev/null +++ b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go @@ -0,0 +1,123 @@ +package newrelicmobileapptoken + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + // US region keys start with AA, followed by a 40 characters hexadecimal string, end with "-NRMA" + // EU region keys start with eu01xx, followed by a 36 characters hexadecimal string, end with "-NRMA" + keyPat = regexp.MustCompile(`\b((AA[0-9a-f]{40}|eu01xx[0-9a-f]{36})-NRMA)\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"-nrma"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicMobileAppToken +} + +func (s Scanner) Description() string { + return "A New Relic Mobile App Token is an authentication key used to send mobile application telemetry data (such as performance metrics, crashes, and events) from iOS and Android apps to New Relic for monitoring and analysis. It is specific to each mobile app and ensures secure data ingestion." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + Redacted: resMatch[:8] + "...", + SecretParts: map[string]string{"key": resMatch}, + } + + if strings.HasPrefix(resMatch, "eu01xx") { + s1.SecretParts["region"] = "eu" + s1.ExtraData = map[string]string{"region": "eu"} + } else { + s1.SecretParts["region"] = "us" + s1.ExtraData = map[string]string{"region": "us"} + } + + if verify { + isVerified, verificationErr := s.verify(ctx, resMatch, s1.SecretParts["region"]) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, nil +} + +// verify checks if the provided key is valid by making a request to the New Relic Android Agent internal API. +// A POST request is made to the /mobile/v5/connect endpoint. If the response status code is 400, +// it indicates that the key is valid but the request is malformed (since we're not sending a proper payload), +// while a 401 status code indicates that the key is invalid. Any other status code is treated as an error. +// This API is not documented, and was discovered by digging into New Relic's Android agent SDK code: +// https://github.com/newrelic/newrelic-android-agent +func (s Scanner) verify(ctx context.Context, key, region string) (bool, error) { + host := "https://mobile-collector.newrelic.com" + + if region == "eu" { + // EU region keys have a different host + host = "https://mobile-collector.eu01.nr-data.net" + } + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, host+"/mobile/v5/connect", http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-App-License-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusBadRequest: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go new file mode 100644 index 000000000000..17ad45605b2f --- /dev/null +++ b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go @@ -0,0 +1,163 @@ +//go:build detectors +// +build detectors + +package newrelicmobileapptoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicMobileAppToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_MOBILE_APP_TOKEN") + keyEU := testSecrets.MustGetField("NEW_RELIC_MOBILE_APP_TOKEN_EU") + keyInactive := "AAcc7eb96551e8cd65818865695f35bb109455d623-NRMA" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic mobile app token %s within", key)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicMobileAppToken, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": key, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic mobile app token %s within", keyEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicMobileAppToken, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + SecretParts: map[string]string{ + "key": keyEU, + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic mobile app token %s within", keyInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicMobileAppToken, + Verified: false, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": keyInactive, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicMobileAppToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicMobileAppToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go new file mode 100644 index 000000000000..4edd199f2389 --- /dev/null +++ b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go @@ -0,0 +1,86 @@ +package newrelicmobileapptoken + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "AAcc7eb96551e8cd65818865695f35bb109455d623-NRMA" + validPatternEU = "eu01xxbfd4a807e4099453ba160493119a126319cb-NRMA" + invalidPattern = "AAcc7eb96551e8cd65818865695f35bb109455d6-NRMA" +) + +func TestNewRelicMobileAppToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic mobile app token = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern EU", + input: fmt.Sprintf("new relic mobile app token EU = '%s'", validPatternEU), + want: []string{validPatternEU}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic mobile app token = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey.go b/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey.go index 56dc90e458de..cea6dd87c4cf 100644 --- a/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey.go +++ b/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey.go @@ -58,12 +58,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result resEU, errEU := client.Do(reqEU) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } } else if errEU == nil { - defer resEU.Body.Close() + defer func() { _ = resEU.Body.Close() }() if resEU.StatusCode >= 200 && resEU.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/newrelicuserkey/newrelicuserkey.go b/pkg/detectors/newrelicuserkey/newrelicuserkey.go new file mode 100644 index 000000000000..6ce52995a1e2 --- /dev/null +++ b/pkg/detectors/newrelicuserkey/newrelicuserkey.go @@ -0,0 +1,147 @@ +package newrelicuserkey + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(`\b(NRAK-[A-Z0-9]{27})\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"nrak-"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicUserKey +} + +func (s Scanner) Description() string { + return "A New Relic User API Key is an authentication token used to query data from New Relic via the NerdGraph API or REST API, allowing users to access account data and perform read operations securely. It is primarily used for interacting with New Relic’s query and configuration services." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + Redacted: resMatch[:8] + "...", + SecretParts: map[string]string{ + "key": resMatch, + }, + } + + if verify { + isVerified, extraData, verificationErr := s.verify(ctx, resMatch) + s1.Verified = isVerified + s1.ExtraData = extraData + if extraData != nil { + s1.SecretParts["region"] = extraData["region"] + } + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, nil +} + +type graphqlResponse struct { + Data struct { + RequestContext struct { + UserID string `json:"userId"` + } `json:"requestContext"` + } `json:"data"` +} + +// verify checks if the provided key is valid by making a request to the New Relic NerdGraph API. +// It sends a POST request to the NerdGraph API. A valid key will result in a 200 OK response. +// Invalid key will return in 401 Unauthorized, and a key with incorrect region will return a 403 Forbidden. +// https://docs.newrelic.com/docs/apis/nerdgraph/get-started/introduction-new-relic-nerdgraph/ +func (s Scanner) verify(ctx context.Context, key string) (bool, map[string]string, error) { + regionUrls := map[string]string{ + "us": "https://api.newrelic.com/graphql", + "eu": "https://api.eu.newrelic.com/graphql", + } + + errs := make([]error, 0, len(regionUrls)) + for region, regionUrl := range regionUrls { + verified, extraData, err := s.verifyRegion(ctx, key, region, regionUrl) + if err != nil { + errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err)) + continue + } + if verified { + return true, extraData, nil + } + } + return false, nil, errors.Join(errs...) +} + +func (s Scanner) verifyRegion(ctx context.Context, key, region, regionUrl string) (bool, map[string]string, error) { + body := `{"query": "{ requestContext { userId } }"}` + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, regionUrl, strings.NewReader(body)) + if err != nil { + return false, nil, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("API-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, nil, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + var resp graphqlResponse + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + return false, nil, fmt.Errorf("error decoding response for region %s: %w", region, err) + } + return true, map[string]string{"region": region, "user_id": resp.Data.RequestContext.UserID}, nil + case http.StatusUnauthorized, http.StatusForbidden: + // 401 means the key is invalid, 403 means the region is incorrect + return false, nil, nil + default: + return false, nil, fmt.Errorf("unexpected status code for region %s: %d", region, res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicuserkey/newrelicuserkey_integration_test.go b/pkg/detectors/newrelicuserkey/newrelicuserkey_integration_test.go new file mode 100644 index 000000000000..6ed84ef75232 --- /dev/null +++ b/pkg/detectors/newrelicuserkey/newrelicuserkey_integration_test.go @@ -0,0 +1,161 @@ +//go:build detectors +// +build detectors + +package newrelicuserkey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicUserKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_USER_KEY") + keyEU := testSecrets.MustGetField("NEW_RELIC_USER_KEY_EU") + keyInactive := "NRAK-GDJSS4QZORUYIC1OGTRNGQMT5VH" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic user key %s within", key)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicUserKey, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + "user_id": "1007651396", + }, + SecretParts: map[string]string{ + "key": key, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic user key %s within", keyEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicUserKey, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + "user_id": "1007651478", + }, + SecretParts: map[string]string{ + "key": keyEU, + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic user key %s within", keyInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicUserKey, + Verified: false, + SecretParts: map[string]string{ + "key": keyInactive, + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicUserKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicUserKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicuserkey/newrelicuserkey_test.go b/pkg/detectors/newrelicuserkey/newrelicuserkey_test.go new file mode 100644 index 000000000000..0659cdf07589 --- /dev/null +++ b/pkg/detectors/newrelicuserkey/newrelicuserkey_test.go @@ -0,0 +1,80 @@ +package newrelicuserkey + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "NRAK-GDJSS4QZORUYIC1OGTRNGQMT5VH" + invalidPattern = "NRAK-GDJSS4QZORUYIC1OGTRNGQMT5V" +) + +func TestNewRelicUserKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic user key = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic user key = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/newsapi/newsapi.go b/pkg/detectors/newsapi/newsapi.go index b928f01419bf..1ab3dee1fdf8 100644 --- a/pkg/detectors/newsapi/newsapi.go +++ b/pkg/detectors/newsapi/newsapi.go @@ -2,10 +2,11 @@ package newsapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/newscatcher/newscatcher.go b/pkg/detectors/newscatcher/newscatcher.go index 47a7bdd6d21d..3ff0e0c7a80f 100644 --- a/pkg/detectors/newscatcher/newscatcher.go +++ b/pkg/detectors/newscatcher/newscatcher.go @@ -2,10 +2,11 @@ package newscatcher import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nexmoapikey/nexmoapikey.go b/pkg/detectors/nexmoapikey/nexmoapikey.go index a3969ac53d0c..1cc0bd50640f 100644 --- a/pkg/detectors/nexmoapikey/nexmoapikey.go +++ b/pkg/detectors/nexmoapikey/nexmoapikey.go @@ -2,10 +2,11 @@ package nexmoapikey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -61,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nftport/nftport.go b/pkg/detectors/nftport/nftport.go index 155cd217dcb0..0dd666d0acde 100644 --- a/pkg/detectors/nftport/nftport.go +++ b/pkg/detectors/nftport/nftport.go @@ -2,10 +2,11 @@ package nftport import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ngc/ngc.go b/pkg/detectors/ngc/ngc.go index 79b1b2f12938..9cd1e1644d78 100644 --- a/pkg/detectors/ngc/ngc.go +++ b/pkg/detectors/ngc/ngc.go @@ -3,10 +3,11 @@ package ngc import ( "context" "encoding/base64" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -63,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ngrok/ngrok.go b/pkg/detectors/ngrok/ngrok.go index d653aac45d2d..0ff517fc0581 100644 --- a/pkg/detectors/ngrok/ngrok.go +++ b/pkg/detectors/ngrok/ngrok.go @@ -38,7 +38,13 @@ const ( ) var ( - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ngrok"}) + `\b(2[a-zA-Z0-9]{26}_\d[a-zA-Z0-9]{20})\b`) + // ngrok API keys and authtokens are {prefix}_{suffix}, both alphanumeric with + // no fixed leading character. The prefix is 27 chars. Suffixes are usually 21 + // chars with a leading digit, but real authtokens verified against the ngrok + // API (via ERR_NGROK_206) have been observed with 20-char and letter-leading + // suffixes, so the pattern must accept 20-21 chars starting with any + // alphanumeric character. + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ngrok"}) + `\b([a-zA-Z0-9]{27}_[a-zA-Z0-9]{20,21})\b`) ) func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { @@ -57,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result r := detectors.Result{ DetectorType: detector_typepb.DetectorType_Ngrok, Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } if verify { @@ -66,9 +73,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, vErr := verifyMatch(ctx, s.client, token) r.Verified = isVerified r.SetVerificationError(vErr, token) - if isVerified { - r.SecretParts = map[string]string{"key": token} - } } results = append(results, r) diff --git a/pkg/detectors/ngrok/ngrok_test.go b/pkg/detectors/ngrok/ngrok_test.go index 033b56ed17da..71d3da08fdee 100644 --- a/pkg/detectors/ngrok/ngrok_test.go +++ b/pkg/detectors/ngrok/ngrok_test.go @@ -12,9 +12,15 @@ import ( ) var ( - validPattern = "2WIRSIHOQyHSVklZnoz2k6bTYdH_0E7z0Ta9QEyR1fZvQ0KU9" - invalidPattern = "2WIRSIHOQyHSVklZnoz2k6bT?dH_0E7z0Ta9QEyR1fZvQ0KU9" - keyword = "ngrok" + validPattern = "2WIRSIHOQyHSVklZnoz2k6bTYdH_0E7z0Ta9QEyR1fZvQ0KU9" + validPatternAlt = "3GuTI5JXfaLtYILkc6N0xopP2V0_89zkDfefiuMv6Hk1z6oYi" + // Authtokens exist in the wild with letter-leading and 20-char suffixes + // (confirmed live against the ngrok API), so both must be detected. + validPatternLetterSuffix = "3GuTI5JXfaLtYILkc6N0xopP2V0_a9zkDfefiuMv6Hk1z6oYi" + validPatternShortSuffix = "4HvUJ6KYgbMuZJMld7O1yqqQ3W1_b8ylEgfgjvNw7Il2z7pZ" + invalidPattern = "2WIRSIHOQyHSVklZnoz2k6bT?dH_0E7z0Ta9QEyR1fZvQ0KU9" + invalidIDPattern = "ak_3GuTI5JXfaLtYILkc6N0xopP2V0" + keyword = "ngrok" ) func TestNgrok_Pattern(t *testing.T) { @@ -30,11 +36,26 @@ func TestNgrok_Pattern(t *testing.T) { input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), want: []string{"2WIRSIHOQyHSVklZnoz2k6bTYdH_0E7z0Ta9QEyR1fZvQ0KU9"}, }, + { + name: "valid pattern - api key not starting with 2", + input: fmt.Sprintf("%s token = '%s'", keyword, validPatternAlt), + want: []string{validPatternAlt}, + }, { name: "valid pattern - ignore duplicate", input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), want: []string{"2WIRSIHOQyHSVklZnoz2k6bTYdH_0E7z0Ta9QEyR1fZvQ0KU9"}, }, + { + name: "valid pattern - authtoken suffix not starting with digit", + input: fmt.Sprintf("%s token = '%s'", keyword, validPatternLetterSuffix), + want: []string{validPatternLetterSuffix}, + }, + { + name: "valid pattern - authtoken with 20 char suffix", + input: fmt.Sprintf("%s token = '%s'", keyword, validPatternShortSuffix), + want: []string{validPatternShortSuffix}, + }, { name: "valid pattern - key out of prefix range", input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern), @@ -45,6 +66,11 @@ func TestNgrok_Pattern(t *testing.T) { input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), want: []string{}, }, + { + name: "invalid pattern - api key resource id not bearer token", + input: fmt.Sprintf("%s api_key = '%s'", keyword, invalidIDPattern), + want: []string{}, + }, } for _, test := range tests { diff --git a/pkg/detectors/nicereply/nicereply.go b/pkg/detectors/nicereply/nicereply.go index f70b5875ba4a..6ee0917de462 100644 --- a/pkg/detectors/nicereply/nicereply.go +++ b/pkg/detectors/nicereply/nicereply.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nightfall/nightfall.go b/pkg/detectors/nightfall/nightfall.go index 6e0cfface20a..382b5e4708fa 100644 --- a/pkg/detectors/nightfall/nightfall.go +++ b/pkg/detectors/nightfall/nightfall.go @@ -3,10 +3,11 @@ package nightfall import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nimble/nimble.go b/pkg/detectors/nimble/nimble.go index 7b85199941cf..5102b933b254 100644 --- a/pkg/detectors/nimble/nimble.go +++ b/pkg/detectors/nimble/nimble.go @@ -3,10 +3,11 @@ package nimble import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/noticeable/noticeable.go b/pkg/detectors/noticeable/noticeable.go index 505ff9b67f8d..0cbda5544b09 100644 --- a/pkg/detectors/noticeable/noticeable.go +++ b/pkg/detectors/noticeable/noticeable.go @@ -3,10 +3,11 @@ package noticeable import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Apikey %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/notion/notion.go b/pkg/detectors/notion/notion.go index e7a2d751deea..c53c42f40731 100644 --- a/pkg/detectors/notion/notion.go +++ b/pkg/detectors/notion/notion.go @@ -42,6 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Notion, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -53,14 +54,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 { // if >= 200 and < 300, the secret is valid and has privileges for the /v1/users endpoint // If 403, the secret is valid, but does not have privileges for the /v1/users endpoint, // Notion returns 401 for all non-valid keys, thus 403 indicates it has fine-tuned permissions, // /v1/search, /v1/databases/*, etc. may work. s1.Verified = true - s1.SecretParts = map[string]string{"key": resMatch} } } else { diff --git a/pkg/detectors/nozbeteams/nozbeteams.go b/pkg/detectors/nozbeteams/nozbeteams.go index c30252e530ac..33edc2dcd5f6 100644 --- a/pkg/detectors/nozbeteams/nozbeteams.go +++ b/pkg/detectors/nozbeteams/nozbeteams.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/npmtoken/npmtoken.go b/pkg/detectors/npmtoken/npmtoken.go index 9f4e957509b2..f50beb993b8f 100644 --- a/pkg/detectors/npmtoken/npmtoken.go +++ b/pkg/detectors/npmtoken/npmtoken.go @@ -44,6 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_NpmToken, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/npm/", @@ -57,12 +58,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true - s1.SecretParts = map[string]string{ - "key": resMatch, - } } } } diff --git a/pkg/detectors/npmtokenv2/npmtokenv2.go b/pkg/detectors/npmtokenv2/npmtokenv2.go index 1a6e36fc4c6d..ca0eb5a0e29a 100644 --- a/pkg/detectors/npmtokenv2/npmtokenv2.go +++ b/pkg/detectors/npmtokenv2/npmtokenv2.go @@ -45,6 +45,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_NpmToken, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/npm/", @@ -58,12 +59,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true - s1.SecretParts = map[string]string{ - "key": resMatch, - } } } } diff --git a/pkg/detectors/nugetapikey/nugetapikey.go b/pkg/detectors/nugetapikey/nugetapikey.go index 19f8bba82c5a..0340bc006c71 100644 --- a/pkg/detectors/nugetapikey/nugetapikey.go +++ b/pkg/detectors/nugetapikey/nugetapikey.go @@ -2,10 +2,11 @@ package nugetapikey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-NuGet-ApiKey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() // we can either match on response code "400" or "Bad Request" response if res.StatusCode == 400 { s1.Verified = true diff --git a/pkg/detectors/numverify/numverify.go b/pkg/detectors/numverify/numverify.go index 4d907c1919dd..0b7793144b16 100644 --- a/pkg/detectors/numverify/numverify.go +++ b/pkg/detectors/numverify/numverify.go @@ -3,11 +3,12 @@ package numverify import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `country_code`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/nutritionix/nutritionix.go b/pkg/detectors/nutritionix/nutritionix.go index 086c4392511e..e70c55c84fcc 100644 --- a/pkg/detectors/nutritionix/nutritionix.go +++ b/pkg/detectors/nutritionix/nutritionix.go @@ -2,10 +2,11 @@ package nutritionix import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -63,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-app-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/nylas/nylas.go b/pkg/detectors/nylas/nylas.go index 8c779901aba4..c6579b1c6514 100644 --- a/pkg/detectors/nylas/nylas.go +++ b/pkg/detectors/nylas/nylas.go @@ -3,10 +3,11 @@ package nylas import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/oanda/oanda.go b/pkg/detectors/oanda/oanda.go index 3715d8444730..6eabcafdd87a 100644 --- a/pkg/detectors/oanda/oanda.go +++ b/pkg/detectors/oanda/oanda.go @@ -3,10 +3,11 @@ package oanda import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/octopusdeploy/octopusdeploy.go b/pkg/detectors/octopusdeploy/octopusdeploy.go new file mode 100644 index 000000000000..d57a9e682ddb --- /dev/null +++ b/pkg/detectors/octopusdeploy/octopusdeploy.go @@ -0,0 +1,154 @@ +package octopusdeploy + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = detectors.NewClientWithDedup(detectors.DetectorHttpClientWithNoLocalAddresses) + + // Octopus Deploy API keys: + // Format: API- followed by 29–34 uppercase letters or digits + octopusTokenPat = regexp.MustCompile( + `\b(API-[A-Z0-9]{29,34})\b`, + ) + + urlPat = regexp.MustCompile(`\b([a-z0-9-]+\.octopus\.app)\b`) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{"octopus"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for Octopus API tokens and optionally verifies them +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + uniqueUrls := make(map[string]struct{}) + + for _, urlMatch := range urlPat.FindAllStringSubmatch(dataStr, -1) { + uniqueUrls[urlMatch[1]] = struct{}{} + } + for _, match := range octopusTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + for url := range uniqueUrls { + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_OctopusDeploy, + Raw: []byte(token), + RawV2: []byte(fmt.Sprintf("%s:%s", url, token)), + Redacted: token[:8] + "...", + SecretParts: map[string]string{ + "key": token, + "url": url, + }, + } + + if verify { + verified, verificationErr := verifyOctopusToken( + ctx, + s.getClient(), + url, + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + } + + return +} + +func verifyOctopusToken( + ctx context.Context, + client *http.Client, + baseUrl string, + token string, +) (bool, error) { + // API REFERENCE: https://trufflesec.octopus.app/api + // DOCS: https://octopus.com/docs/octopus-rest-api + url := &url.URL{ + Scheme: "https", + Host: baseUrl, + Path: "/api/users/me", + } + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + url.String(), + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("X-Octopus-ApiKey", token) + + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_OctopusDeploy, fmt.Sprintf("%s:%s", baseUrl, token), req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK, http.StatusForbidden: + return true, nil + + case http.StatusUnauthorized: + // Invalid or revoked key + return false, nil + + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_OctopusDeploy +} + +func (s Scanner) Description() string { + return "Octopus Deploy is a DevOps deployment automation platform. Octopus Deploy API keys can be used to automate deployments, manage projects, environments, and infrastructure resources." +} diff --git a/pkg/detectors/octopusdeploy/octopusdeploy_integration_test.go b/pkg/detectors/octopusdeploy/octopusdeploy_integration_test.go new file mode 100644 index 000000000000..2d3b6ecb3556 --- /dev/null +++ b/pkg/detectors/octopusdeploy/octopusdeploy_integration_test.go @@ -0,0 +1,206 @@ +//go:build detectors +// +build detectors + +package octopusdeploy + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestOctopusDeploy_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Expect secrets structured like: + // OCTOPUS_CLOUD_URL = acme.octopus.app + // OCTOPUS_API_KEY = API-XXXXXXXXXXXXXXXXXXXXXXXXXXXX + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + baseURL := testSecrets.MustGetField("OCTOPUS_CLOUD_URL") + activeToken := testSecrets.MustGetField("OCTOPUS_API_KEY") + + inactiveToken := "API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Server %s using key %s", + baseURL, + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_OctopusDeploy, + Verified: true, + Raw: []byte(activeToken), + RawV2: []byte(fmt.Sprintf("%s:%s", baseURL, activeToken)), + }, + }, + }, + { + name: "found, real token, verification timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "%s %s", + baseURL, + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_OctopusDeploy, + Verified: false, + Raw: []byte(activeToken), + RawV2: []byte(fmt.Sprintf("%s:%s", baseURL, activeToken)), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, unexpected api response", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "%s %s", + baseURL, + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_OctopusDeploy, + Verified: false, + Raw: []byte(activeToken), + RawV2: []byte(fmt.Sprintf("%s:%s", baseURL, activeToken)), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "%s %s", + baseURL, + inactiveToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_OctopusDeploy, + Verified: false, + Raw: []byte(inactiveToken), + RawV2: []byte(fmt.Sprintf("%s:%s", baseURL, inactiveToken)), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("OctopusDeploy.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "Redacted", + "chunkOffset", + "chunkOffsetSet", + "SecretParts", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("OctopusDeploy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkOctopusDeploy_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/octopusdeploy/octopusdeploy_test.go b/pkg/detectors/octopusdeploy/octopusdeploy_test.go new file mode 100644 index 000000000000..4a5cda9b9164 --- /dev/null +++ b/pkg/detectors/octopusdeploy/octopusdeploy_test.go @@ -0,0 +1,192 @@ +package octopusdeploy + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestOctopusDeploy_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: ` + func setupOctopusClient() (*http.Client, error) { + baseUrl := &url.URL{Scheme: "https", Host: "acme.octopus.app", Path: "/api/users/me"} + + // Create a new request with the API key as a header + req, err := http.NewRequest("GET", baseUrl.String(), http.NoBody) + if err != nil { + return nil, err + } + req.Header.Set("X-Octopus-ApiKey", "API-1234567890ABCDEFGHIJKLMNO1234") + + client := &http.Client{} + resp, _ := client.Do(req) + defer func() { _ = resp.Body.Close() }() + + return client, nil + } + `, + want: []string{ + "acme.octopus.app:API-1234567890ABCDEFGHIJKLMNO1234", + }, + }, + { + name: "valid pattern - env file", + input: ` + # .env.production + APP_NAME=deploy-bot + OCTOPUS_URL=prod.octopus.app + OCTOPUS_API_KEY=API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA + LOG_LEVEL=info + `, + want: []string{ + "prod.octopus.app:API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + func loadOctopusKeys() []string { + // Rotated keys retained during the grace period + return []string{ + "API-11111111111111111111111111111", + "API-22222222222222222222222222222", + } + } + + // deployments run against dev.octopus.app + `, + want: []string{ + "dev.octopus.app:API-11111111111111111111111111111", + "dev.octopus.app:API-22222222222222222222222222222", + }, + }, + { + name: "valid pattern - multiple urls and tokens", + input: ` + environments: + - name: staging + url: acme.octopus.app + - name: production + url: prod.octopus.app + + # shared deployment key used across environments + api_key: API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA + `, + want: []string{ + "acme.octopus.app:API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "prod.octopus.app:API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }, + }, + { + name: "invalid pattern - lowercase token", + input: ` + func setupOctopusClient() { + baseUrl := "acme.octopus.app" + // Key was accidentally lowercased during a config export + apiKey := "API-abcdefghijklmnopqrstuvwxyz1234" + client.SetApiKey(baseUrl, apiKey) + } + `, + want: nil, + }, + { + name: "invalid pattern - too short", + input: ` + func setupOctopusClient() { + baseUrl := "acme.octopus.app" + // Truncated key accidentally committed + apiKey := "API-1234" + client.SetApiKey(baseUrl, apiKey) + } + `, + want: nil, + }, + { + name: "invalid pattern - too long", + input: ` + func setupOctopusClient() { + baseUrl := "acme.octopus.app" + // Extra characters appended by a bad find-and-replace + apiKey := "API-ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" + client.SetApiKey(baseUrl, apiKey) + } + `, + want: nil, + }, + { + name: "invalid pattern - url only", + input: ` + func octopusHealthCheck() error { + resp, err := http.Get("https://acme.octopus.app/api/status") + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + return nil + } + `, + want: nil, + }, + { + name: "invalid pattern - token only", + input: ` + // TODO: load the real key instead of octopus_token env var + func isOctopusToken(s string) bool { + return strings.HasPrefix(s, "API-AAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + } + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords()) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + actual[string(r.RawV2)] = struct{}{} + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go index 56b7bad686fe..bdabc153ad98 100644 --- a/pkg/detectors/okta/okta.go +++ b/pkg/detectors/okta/okta.go @@ -20,12 +20,26 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.CustomResultsCleaner = (*Scanner)(nil) var ( defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`) tokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`) - // TODO: Oauth client secrets + + // Okta tenant domains: okta.com, oktapreview.com, okta-emea.com, okta-gov.com + // (FedRAMP), and okta.mil (US Military). Every finding pairs with a matched + // domain, so a suffix missing here means zero findings for that tenant. + // Custom URL domains (e.g. login.example.com) are deliberately not matched: + // they are indistinguishable from any other hostname, and guessing would send + // credentials to unvetted hosts during verification. Such tenants are still + // found when their original *.okta.com hostname appears in the chunk. + domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.(?:okta(?:preview|-emea|-gov)?\.com|okta\.mil)\b`) + + // Okta app/client IDs: "0oa" + 17 base62 characters. + oauthClientIDPat = regexp.MustCompile(`\b0oa[a-zA-Z0-9]{17}\b`) + // Delimiters consume a char instead of using \b, since \b breaks on secrets + // starting/ending in '-' (a non-word char that's also part of the charset). + oauthClientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"client_secret", "clientsecret", "client-secret"}) + `(?:^|[^a-zA-Z0-9_-])([a-zA-Z0-9_-]{40,64})(?:[^a-zA-Z0-9_-]|$)`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -38,7 +52,10 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - var uniqueTokens, uniqueDomains = make(map[string]struct{}), make(map[string]struct{}) + uniqueTokens := make(map[string]struct{}) + uniqueDomains := make(map[string]struct{}) + uniqueClientIDs := make(map[string]struct{}) + uniqueClientSecrets := make(map[string]struct{}) for _, matches := range tokenPat.FindAllStringSubmatch(dataStr, -1) { uniqueTokens[matches[0]] = struct{}{} @@ -48,11 +65,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result uniqueDomains[matches[0]] = struct{}{} } + for _, matches := range oauthClientIDPat.FindAllStringSubmatch(dataStr, -1) { + uniqueClientIDs[matches[0]] = struct{}{} + } + + for _, matches := range oauthClientSecretPat.FindAllStringSubmatch(dataStr, -1) { + uniqueClientSecrets[matches[1]] = struct{}{} + } + for token := range uniqueTokens { for domain := range uniqueDomains { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Okta, Raw: []byte(token), + Redacted: fmt.Sprintf("%s:%s...%s", domain, token[:4], token[len(token)-4:]), SecretParts: map[string]string{ "domain": domain, "token": token, @@ -75,6 +101,37 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } } + for clientID := range uniqueClientIDs { + for clientSecret := range uniqueClientSecrets { + for domain := range uniqueDomains { + s2 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_Okta, + Raw: []byte(clientSecret), + Redacted: fmt.Sprintf("%s:%s", domain, clientID), + SecretParts: map[string]string{ + "domain": domain, + "client_id": clientID, + "client_secret": clientSecret, + }, + RawV2: []byte(fmt.Sprintf("%s:%s:%s", domain, clientID, clientSecret)), + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + + isVerified, verificationErr := verifyOktaOAuthClientCredentials(ctx, client, domain, clientID, clientSecret) + s2.Verified = isVerified + s2.SetVerificationError(verificationErr) + } + + results = append(results, s2) + } + } + } + return } @@ -98,7 +155,7 @@ func verifyOktaToken(ctx context.Context, client *http.Client, domain, token str if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: @@ -111,6 +168,89 @@ func verifyOktaToken(ctx context.Context, client *http.Client, domain, token str } } +func verifyOktaOAuthClientCredentials(ctx context.Context, client *http.Client, domain, clientID, clientSecret string) (bool, error) { + // curl -v -X POST \ + // -H "Accept: application/json" \ + // -H "Content-Type: application/x-www-form-urlencoded" \ + // -u "clientID:clientSecret" \ + // -d "grant_type=client_credentials" \ + // "https://subdomain.okta.com/oauth2/default/v1/token" + + const body = "grant_type=client_credentials" + + url := fmt.Sprintf("https://%s/oauth2/default/v1/token", domain) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body)) + if err != nil { + return false, err + } + req.SetBasicAuth(clientID, clientSecret) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = resp.Body.Close() }() + + switch resp.StatusCode { + case http.StatusOK: + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + return strings.Contains(string(respBody), "\"access_token\""), nil + case http.StatusBadRequest: + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + // Okta authenticates the client before grant-type checks, so "unauthorized_client" still confirms the secret. + if strings.Contains(string(respBody), "unauthorized_client") { + return true, nil + } + if strings.Contains(string(respBody), "invalid_client") { + return false, nil + } + // Other 400s (e.g. invalid_scope) say nothing about the secret; surface as an error. + return false, fmt.Errorf("response body missing expected keyword") + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } +} + +// CleanResults keeps the best (preferably verified) result per distinct secret +// identity (Redacted), rather than collapsing unrelated SSWS and OAuth findings +// down to a single arbitrary result the way detectors.CleanResults would. +func (s Scanner) CleanResults(results []detectors.Result, _ bool) []detectors.Result { + if len(results) == 0 { + return results + } + + byIdentity := make(map[string]detectors.Result, len(results)) + for _, r := range results { + if r.Verified { + byIdentity[r.Redacted] = r + continue + } + if _, exists := byIdentity[r.Redacted]; !exists { + byIdentity[r.Redacted] = r + } + } + + cleaned := make([]detectors.Result, 0, len(byIdentity)) + for _, r := range byIdentity { + cleaned = append(cleaned, r) + } + return cleaned +} + +func (s Scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool { + return false +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Okta } diff --git a/pkg/detectors/okta/okta_test.go b/pkg/detectors/okta/okta_test.go index c8fc81a617f7..ce0972ac6400 100644 --- a/pkg/detectors/okta/okta_test.go +++ b/pkg/detectors/okta/okta_test.go @@ -7,16 +7,37 @@ import ( "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) var ( - validDomain = "a254u0kjk-y64jh762weozo374t.okta.com" - invalidDomain = "a254u0kjk-y64jh762?eozo374t.okta.com" - validToken = "00JIBRGk12UbkjQaaw4L7VYy9EE8zAEEkIeqGNcSQm" - invalidToken = "00JIBRGk12UbkjQaaw4L7?Yy9EE8zAEEkIeqGNcSQm" - keyword = "okta" + validDomain = "a254u0kjk-y64jh762weozo374t.okta.com" + invalidDomain = "a254u0kjk-y64jh762?eozo374t.okta.com" + validToken = "00JIBRGk12UbkjQaaw4L7VYy9EE8zAEEkIeqGNcSQm" + invalidToken = "00JIBRGk12UbkjQaaw4L7?Yy9EE8zAEEkIeqGNcSQm" + validClientID = "0oakDa9U4UqGWlG6g3Ot" + invalidClientID = "0obkDa9U4UqGWlG6g3Ot" + validClientSecret = "OhbVrpoiVgRV5IfLBcbfnoGMbJmTPSIAoCLrZ3aW" + invalidClientSecret = "OhbVrpoiVgRV5IfLBcbfnoGMbJmTPSIAoCLrZ3a?" + // Regression fixtures for a boundary bug where `\b` around a charset that includes + // '-' either failed to match, or silently dropped a leading/trailing hyphen. + leadingHyphenSecret40 = "-u8jzPde0IgxLd6GncfBAepfJBd0Kh8oOOL8dKLz" + trailingHyphenSecret40 = "u8jzPde0IgxLd6GncfBAepfJBd0Kh8oOOL8dKLz-" + leadingHyphenSecret41 = "-u8jzPde0IgxLd6GncfBAepfJBd0Kh8oOOL8dKLzd" + trailingHyphenSecret41 = "u8jzPde0IgxLd6GncfBAepfJBd0Kh8oOOL8dKLzd-" + keyword = "okta" + + // One fixture per Okta tenant suffix, plus near-misses that must not match. + previewDomain = "acmecorp.oktapreview.com" + emeaDomain = "acmecorp.okta-emea.com" + govDomain = "acmecorp.okta-gov.com" + milDomain = "acmecorp.okta.mil" + // okta-dnssec.com is a CNAME target for custom-domain DNSSEC, not a tenant domain. + dnssecDomain = "acmecorp.okta-dnssec.com" + // Guards the word boundary after the "mil" suffix. + militaryDomain = "acmecorp.okta.military" ) func TestOkta_Pattern(t *testing.T) { @@ -37,6 +58,76 @@ func TestOkta_Pattern(t *testing.T) { input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidDomain, keyword, invalidToken), want: []string{}, }, + { + name: "valid pattern - oauth client credentials", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, validClientID, validClientSecret), + want: []string{validDomain + ":" + validClientID + ":" + validClientSecret}, + }, + { + name: "invalid pattern - oauth client credentials", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, invalidClientID, invalidClientSecret), + want: []string{}, + }, + { + name: "valid pattern - oauth client_secret (40 chars) starting with hyphen", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, validClientID, leadingHyphenSecret40), + want: []string{validDomain + ":" + validClientID + ":" + leadingHyphenSecret40}, + }, + { + name: "valid pattern - oauth client_secret (40 chars) ending with hyphen", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, validClientID, trailingHyphenSecret40), + want: []string{validDomain + ":" + validClientID + ":" + trailingHyphenSecret40}, + }, + { + name: "valid pattern - oauth client_secret (41 chars) starting with hyphen", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, validClientID, leadingHyphenSecret41), + want: []string{validDomain + ":" + validClientID + ":" + leadingHyphenSecret41}, + }, + { + name: "valid pattern - oauth client_secret (41 chars) ending with hyphen", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, validClientID, trailingHyphenSecret41), + want: []string{validDomain + ":" + validClientID + ":" + trailingHyphenSecret41}, + }, + { + name: "valid pattern - token with oktapreview.com domain", + input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, previewDomain, keyword, validToken), + want: []string{previewDomain + ":" + validToken}, + }, + { + name: "valid pattern - token with okta-emea.com domain", + input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, emeaDomain, keyword, validToken), + want: []string{emeaDomain + ":" + validToken}, + }, + { + name: "valid pattern - token with okta-gov.com domain", + input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, govDomain, keyword, validToken), + want: []string{govDomain + ":" + validToken}, + }, + { + name: "valid pattern - token with okta.mil domain", + input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, milDomain, keyword, validToken), + want: []string{milDomain + ":" + validToken}, + }, + { + name: "valid pattern - oauth client credentials with okta-gov.com domain", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, govDomain, keyword, validClientID, validClientSecret), + want: []string{govDomain + ":" + validClientID + ":" + validClientSecret}, + }, + { + name: "valid pattern - oauth client credentials with okta.mil domain", + input: fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, milDomain, keyword, validClientID, validClientSecret), + want: []string{milDomain + ":" + validClientID + ":" + validClientSecret}, + }, + { + name: "invalid pattern - okta-dnssec.com is not a tenant domain", + input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, dnssecDomain, keyword, validToken), + want: []string{}, + }, + { + name: "invalid pattern - okta.mil suffix must end at a word boundary", + input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, militaryDomain, keyword, validToken), + want: []string{}, + }, } for _, test := range tests { @@ -81,3 +172,139 @@ func TestOkta_Pattern(t *testing.T) { }) } } + +func TestOkta_OAuthVerification(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantVerified bool + wantErr bool + }{ + { + name: "200 OK with access token is verified", + statusCode: 200, + body: `{"access_token":"eyJhbGciOiJIUzI1NiJ9...","token_type":"Bearer","expires_in":3600}`, + wantVerified: true, + }, + { + name: "400 unauthorized_client (valid secret, grant type not enabled) is verified", + statusCode: 400, + body: `{"error":"unauthorized_client","error_description":"The client is not authorized to use the provided grant type."}`, + wantVerified: true, + }, + { + name: "400 invalid_client (wrong secret) is unverified", + statusCode: 400, + body: `{"error":"invalid_client","error_description":"The client secret supplied for a confidential client is invalid."}`, + wantVerified: false, + }, + { + name: "400 invalid_client (nonexistent client id) is unverified", + statusCode: 400, + body: `{"errorCode":"invalid_client","errorSummary":"Invalid value for 'client_id' parameter."}`, + wantVerified: false, + }, + { + name: "400 without recognized keyword (e.g. invalid_scope) returns verification error", + statusCode: 400, + body: `{"error":"invalid_scope","error_description":"The authorization server resource does not have any configured default scopes."}`, + wantVerified: false, + wantErr: true, + }, + { + name: "401 is unverified", + statusCode: 401, + body: ``, + wantVerified: false, + }, + { + name: "unexpected status code returns verification error", + statusCode: 500, + body: ``, + wantVerified: false, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := Scanner{client: common.ConstantResponseHttpClient(test.statusCode, test.body)} + data := []byte(fmt.Sprintf("%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", keyword, validDomain, keyword, validClientID, validClientSecret)) + + results, err := s.FromData(context.Background(), true, data) + if err != nil { + t.Fatalf("FromData() unexpected error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + if results[0].Verified != test.wantVerified { + t.Errorf("Verified = %v, want %v", results[0].Verified, test.wantVerified) + } + if gotErr := results[0].VerificationError() != nil; gotErr != test.wantErr { + t.Errorf("verification error present = %v, want %v (err = %v)", gotErr, test.wantErr, results[0].VerificationError()) + } + }) + } +} + +// TestOkta_CleanResults is a regression test: the generic detectors.CleanResults +// collapses an entire result set down to results[:1] whenever nothing verifies, which +// would silently drop a real OAuth client_secret finding in favor of an unrelated +// unverified SSWS-token finding from the same chunk (or vice versa). Okta's custom +// CleanResults must instead keep one result per distinct secret identity (Redacted). +func TestOkta_CleanResults(t *testing.T) { + s := Scanner{} + + t.Run("distinct unverified findings both survive", func(t *testing.T) { + tokenResult := detectors.Result{Redacted: "domain:token-identity", Raw: []byte(validToken)} + oauthResult := detectors.Result{Redacted: "domain:client-id-identity", Raw: []byte(validClientSecret)} + + cleaned := s.CleanResults([]detectors.Result{tokenResult, oauthResult}, true) + if len(cleaned) != 2 { + t.Fatalf("expected both distinct findings to survive cleaning, got %d: %+v", len(cleaned), cleaned) + } + }) + + t.Run("verified duplicate wins over unverified duplicate with the same identity", func(t *testing.T) { + unverifiedDup := detectors.Result{Redacted: "same-identity", Verified: false, Raw: []byte("a")} + verifiedDup := detectors.Result{Redacted: "same-identity", Verified: true, Raw: []byte("b")} + + cleaned := s.CleanResults([]detectors.Result{unverifiedDup, verifiedDup}, true) + if len(cleaned) != 1 || !cleaned[0].Verified { + t.Fatalf("expected the single verified duplicate to win, got %+v", cleaned) + } + }) +} + +// TestOkta_FromData_DistinctRedacted confirms FromData itself assigns non-empty, +// distinct Redacted identities to the token and OAuth result shapes, which is what +// TestOkta_CleanResults relies on to avoid conflating unrelated findings. +func TestOkta_FromData_DistinctRedacted(t *testing.T) { + d := Scanner{} + data := []byte(fmt.Sprintf( + "%s token - '%s'\n%s domain - '%s'\n%s client_id - '%s'\nclient_secret: '%s'\n", + keyword, validToken, keyword, validDomain, keyword, validClientID, validClientSecret, + )) + + results, err := d.FromData(context.Background(), false, data) + if err != nil { + t.Fatalf("FromData() unexpected error = %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results (token + oauth), got %d: %+v", len(results), results) + } + + seen := make(map[string]struct{}, len(results)) + for _, r := range results { + if r.Redacted == "" { + t.Errorf("result has empty Redacted identity: %+v", r) + } + seen[r.Redacted] = struct{}{} + } + if len(seen) != len(results) { + t.Errorf("expected distinct Redacted identities per result, got %v", seen) + } +} diff --git a/pkg/detectors/omnisend/omnisend.go b/pkg/detectors/omnisend/omnisend.go index 70ea501c7900..70f315841b4f 100644 --- a/pkg/detectors/omnisend/omnisend.go +++ b/pkg/detectors/omnisend/omnisend.go @@ -2,10 +2,11 @@ package omnisend import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-KEY", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/onedesk/onedesk.go b/pkg/detectors/onedesk/onedesk.go index 475c0fb8b60a..7c4910884284 100644 --- a/pkg/detectors/onedesk/onedesk.go +++ b/pkg/detectors/onedesk/onedesk.go @@ -57,24 +57,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := strings.NewReader(fmt.Sprintf(`{"email": "%s", "password": "%s"}`, emailMatch, resPword)) - req, err := http.NewRequestWithContext(ctx, "POST", "https://app.onedesk.com/rest/2.0/login/loginUser", payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"code":"SUCCESS"`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, emailMatch, resPword) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, emailMatch, resPword) } results = append(results, s1) @@ -83,6 +68,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, emailMatch string, resPword string) (bool, error) { + payload := strings.NewReader(fmt.Sprintf(`{"email": "%s", "password": "%s"}`, emailMatch, resPword)) + req, err := http.NewRequestWithContext(ctx, "POST", "https://app.onedesk.com/rest/2.0/login/loginUser", payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"code":"SUCCESS"`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Onedesk } diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.go index 64b7a10582f8..699e129fb621 100644 --- a/pkg/detectors/onelogin/onelogin.go +++ b/pkg/detectors/onelogin/onelogin.go @@ -3,11 +3,12 @@ package onelogin import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -65,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json; charset=utf-8") res, err := client.Do(req) if err == nil { - res.Body.Close() // The request body is unused. + _ = res.Body.Close() // The request body is unused. if res.StatusCode >= 200 && res.StatusCode < 300 { result.Verified = true diff --git a/pkg/detectors/onepagecrm/onepagecrm.go b/pkg/detectors/onepagecrm/onepagecrm.go index 536a865d49b2..adbbcc598db0 100644 --- a/pkg/detectors/onepagecrm/onepagecrm.go +++ b/pkg/detectors/onepagecrm/onepagecrm.go @@ -2,10 +2,11 @@ package onepagecrm import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/onesignal/onesignal.go b/pkg/detectors/onesignal/onesignal.go index 839007378651..3843161ce332 100644 --- a/pkg/detectors/onesignal/onesignal.go +++ b/pkg/detectors/onesignal/onesignal.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/oopspam/oopspam.go b/pkg/detectors/oopspam/oopspam.go index 92d398a5e31d..f39759f50d9a 100644 --- a/pkg/detectors/oopspam/oopspam.go +++ b/pkg/detectors/oopspam/oopspam.go @@ -2,10 +2,11 @@ package oopspam import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/openai/openai.go b/pkg/detectors/openai/openai.go index 2fdd52659a66..fb52e065cead 100644 --- a/pkg/detectors/openai/openai.go +++ b/pkg/detectors/openai/openai.go @@ -25,7 +25,14 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + // The OpenAI API can be slow to respond under load, so use a longer + // per-attempt timeout than the default 5s and retry transient failures + // (timeouts, connection errors, 429/5xx) so a single slow response does + // not record an indeterminate verification result. + defaultClient = common.RetryableHTTPClient( + common.WithTimeout(10*time.Second), + common.WithMaxRetries(2), + ) // The magic string T3BlbkFJ is the base64-encoded string: OpenAI // Matches: legacy keys (sk-{alnum}T3BlbkFJ...), project keys (sk-proj-...), @@ -54,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_OpenAI, Redacted: token[:3] + "..." + token[min(len(token)-1, 47):], Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } if verify { @@ -66,7 +74,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = verified s1.ExtraData = extraData s1.SetVerificationError(verificationErr) - s1.SecretParts = map[string]string{"key": token} } results = append(results, s1) diff --git a/pkg/detectors/openai/openai_test.go b/pkg/detectors/openai/openai_test.go index a0cfa8b4157d..98cc737bec48 100644 --- a/pkg/detectors/openai/openai_test.go +++ b/pkg/detectors/openai/openai_test.go @@ -2,12 +2,76 @@ package openai import ( "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "github.com/google/go-cmp/cmp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" - "testing" ) +// The default client must retry transient failures so a single slow or +// failed OpenAI API response does not record an indeterminate verification +// result (CSM-2131). +func TestOpenAI_DefaultClientRetriesTransientErrors(t *testing.T) { + var requests atomic.Int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) < 3 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, ts.URL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + res, err := defaultClient.Do(req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = res.Body.Close() }() + + if res.StatusCode != http.StatusOK { + t.Errorf("expected retries to reach a 200 response, got %d", res.StatusCode) + } + if got := requests.Load(); got != 3 { + t.Errorf("expected 3 attempts (initial + 2 retries), got %d", got) + } +} + +// When the API never recovers, the client must give up after the configured +// retry budget (initial attempt + 2 retries) and surface the failure rather +// than retrying indefinitely. +func TestOpenAI_DefaultClientGivesUpAfterRetryBudget(t *testing.T) { + var requests atomic.Int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer ts.Close() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, ts.URL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + res, err := defaultClient.Do(req) + if res != nil { + defer func() { _ = res.Body.Close() }() + } + + if err == nil && res.StatusCode != http.StatusInternalServerError { + t.Errorf("expected exhausted retries to surface the failure, got status %d with no error", res.StatusCode) + } + if got := requests.Load(); got != 3 { + t.Errorf("expected 3 attempts (initial + 2 retries), got %d", got) + } +} + func TestOpenAI_DoesNotMatchAdminKeys(t *testing.T) { d := Scanner{} adminKey := `OPENAI_ADMIN_KEY = "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"` diff --git a/pkg/detectors/openaiadmin/openaiadmin.go b/pkg/detectors/openaiadmin/openaiadmin.go index f6d959d4f186..9d95b93ba83e 100644 --- a/pkg/detectors/openaiadmin/openaiadmin.go +++ b/pkg/detectors/openaiadmin/openaiadmin.go @@ -48,6 +48,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_OpenAIAdmin, Redacted: token[:11] + "..." + token[len(token)-4:], Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } if verify { @@ -59,9 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyMatch(ctx, client, token) s1.Verified = isVerified s1.SetVerificationError(verificationErr, token) - s1.SecretParts = map[string]string{ - "key": token, - } } results = append(results, s1) diff --git a/pkg/detectors/opencagedata/opencagedata.go b/pkg/detectors/opencagedata/opencagedata.go index 9ca1b56ac600..ba644575f776 100644 --- a/pkg/detectors/opencagedata/opencagedata.go +++ b/pkg/detectors/opencagedata/opencagedata.go @@ -3,10 +3,11 @@ package opencagedata import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_OpenCageData, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/openrouter/openrouter.go b/pkg/detectors/openrouter/openrouter.go new file mode 100644 index 000000000000..cc9b25b19ff2 --- /dev/null +++ b/pkg/detectors/openrouter/openrouter.go @@ -0,0 +1,138 @@ +package openrouter + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + keyPat = regexp.MustCompile(`\b(sk-or-v1-[0-9a-f]{64})\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"sk-or-v1-"} +} + +// FromData will find and optionally verify OpenRouter secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueMatches := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueMatches[match[1]] = struct{}{} + } + + for token := range uniqueMatches { + s1 := detectors.Result{ + DetectorType: s.Type(), + // NOTE: we redact the same way it is done in the `Label` field + Redacted: token[:12] + "..." + token[70:], + Raw: []byte(token), + SecretParts: map[string]string{"key": token}, + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + + verified, extraData, verificationErr := verifyToken(ctx, client, token) + s1.Verified = verified + s1.ExtraData = extraData + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, err +} + +func verifyToken(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://openrouter.ai/api/v1/key", nil) + if err != nil { + return false, nil, err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + var keyResponse keyResponse + if err = json.NewDecoder(res.Body).Decode(&keyResponse); err != nil { + return false, nil, err + } + + key := keyResponse.Data + extraData := map[string]string{ + "label": key.Label, + "limit": fmtFloatPtr(key.Limit), + "usage": fmt.Sprintf("%f", key.Usage), + "is_free_tier": strconv.FormatBool(key.IsFreeTier), + "limit_remaining": fmtFloatPtr(key.LimitRemaining), + } + return true, extraData, nil + case http.StatusUnauthorized: + // Invalid + return false, nil, nil + default: + return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_OpenRouter +} + +func (s Scanner) Description() string { + return "OpenRouter provides a unified API that gives you access to hundreds of AI models through a single endpoint, while automatically handling fallbacks and selecting the most cost-effective options." +} + +func fmtFloatPtr(f *float64) string { + if f == nil { + return "" + } + return fmt.Sprintf("%f", *f) +} + +type keyResponse struct { + Data key `json:"data"` +} + +type key struct { + Label string `json:"label"` + Limit *float64 `json:"limit"` + Usage float64 `json:"usage"` + IsFreeTier bool `json:"is_free_tier"` + LimitRemaining *float64 `json:"limit_remaining"` +} diff --git a/pkg/detectors/openrouter/openrouter_integration_test.go b/pkg/detectors/openrouter/openrouter_integration_test.go new file mode 100644 index 000000000000..7084f3c4e97d --- /dev/null +++ b/pkg/detectors/openrouter/openrouter_integration_test.go @@ -0,0 +1,125 @@ +//go:build detectors +// +build detectors + +package openrouter + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestOpenRouter_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + secret := testSecrets.MustGetField("OPENROUTER") + inactiveSecret := testSecrets.MustGetField("OPENROUTER_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "Found, unverified OpenRouter token sk-or-v1-", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an OpenRouter secret %s within", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_OpenRouter, + Redacted: inactiveSecret[:12] + "..." + inactiveSecret[70:], + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "Found, verified OpenRouter token sk-or-v1-", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an OpenRouter secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_OpenRouter, + Verified: true, + Redacted: secret[:12] + "..." + secret[70:], + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("OpenRouter.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + got[i].Raw = nil + got[i].ExtraData = nil + got[i].SecretParts = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("OpenRouter.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/openrouter/openrouter_test.go b/pkg/detectors/openrouter/openrouter_test.go new file mode 100644 index 000000000000..9e90362153cf --- /dev/null +++ b/pkg/detectors/openrouter/openrouter_test.go @@ -0,0 +1,78 @@ +package openrouter + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestOpenRouter_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "API key", + input: `OPENROUTER_API_KEY = "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"`, + want: []string{"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"}, + }, + { + name: "invalid pattern", + input: ` + [INFO] Sending request to the openrouter API + [DEBUG] Using Key=sk-or-v1-a2Cy8xCLyvrAf7lZKfhQhyCr4RAID9D + [ERROR] Response received: 401 UnAuthorized + `, + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + detectorMatches := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(detectorMatches) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/openuv/openuv.go b/pkg/detectors/openuv/openuv.go index daeaf7be69e6..af0ffbd18840 100644 --- a/pkg/detectors/openuv/openuv.go +++ b/pkg/detectors/openuv/openuv.go @@ -2,10 +2,11 @@ package openuv import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-access-token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/openvpn/openvpn.go b/pkg/detectors/openvpn/openvpn.go index 1feca62e4b78..fc157dfd23a3 100644 --- a/pkg/detectors/openvpn/openvpn.go +++ b/pkg/detectors/openvpn/openvpn.go @@ -81,7 +81,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("content-type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/openweather/openweather.go b/pkg/detectors/openweather/openweather.go index 8d8895dfb435..9ae6567da9c2 100644 --- a/pkg/detectors/openweather/openweather.go +++ b/pkg/detectors/openweather/openweather.go @@ -2,10 +2,11 @@ package openweather import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/opsgenie/opsgenie.go b/pkg/detectors/opsgenie/opsgenie.go index 2d84c1a4e2b1..17918be5edff 100644 --- a/pkg/detectors/opsgenie/opsgenie.go +++ b/pkg/detectors/opsgenie/opsgenie.go @@ -64,6 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result r := detectors.Result{ DetectorType: detector_typepb.DetectorType_Opsgenie, Raw: []byte(key), + SecretParts: map[string]string{"key": key}, } if verify { @@ -76,9 +77,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if isVerified { r.Verified = isVerified r.ExtraData = extraData - r.SecretParts = map[string]string{ - "key": key, - } } r.SetVerificationError(vErr, key) } diff --git a/pkg/detectors/optimizely/optimizely.go b/pkg/detectors/optimizely/optimizely.go index 55827f93ff0f..4392470d8118 100644 --- a/pkg/detectors/optimizely/optimizely.go +++ b/pkg/detectors/optimizely/optimizely.go @@ -3,10 +3,11 @@ package optimizely import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/overloop/overloop.go b/pkg/detectors/overloop/overloop.go index 93e5c7ecd52a..a8e10d0b0cb4 100644 --- a/pkg/detectors/overloop/overloop.go +++ b/pkg/detectors/overloop/overloop.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/owlbot/owlbot.go b/pkg/detectors/owlbot/owlbot.go index 34725279dcf8..95a5840d7b16 100644 --- a/pkg/detectors/owlbot/owlbot.go +++ b/pkg/detectors/owlbot/owlbot.go @@ -3,10 +3,11 @@ package owlbot import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/packagecloud/packagecloud.go b/pkg/detectors/packagecloud/packagecloud.go index 2b5055a469cd..6c619adba708 100644 --- a/pkg/detectors/packagecloud/packagecloud.go +++ b/pkg/detectors/packagecloud/packagecloud.go @@ -2,10 +2,11 @@ package packagecloud import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey.go index 01aefe3ef921..67bcbcd65b03 100644 --- a/pkg/detectors/pagerdutyapikey/pagerdutyapikey.go +++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey.go @@ -80,14 +80,41 @@ func verifyPagerdutyapikey(ctx context.Context, client *http.Client, token strin if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: return true, nil case http.StatusUnauthorized: + // Token is missing, malformed, or invalid. + // Note that revoked (disabled) keys are treated as invalid. + return false, nil + case http.StatusForbidden: + // PagerDuty returns 403 when the token authenticated successfully but + // lacks permission for the requested resource -- e.g. a user token for + // a restricted-role user, or a scoped OAuth token missing users.read. + // This confirms the credential is live. + return true, nil + case http.StatusTooManyRequests: + // Rate-limited. Can't determine validity; keep indeterminate for retry. + return false, fmt.Errorf("PagerDuty rate limit reached") + case http.StatusBadRequest: + // Malformed request parameters. Shouldn't happen for our simple GET. + // Not transient -- the same request will always produce the same result. + return false, nil + case http.StatusPaymentRequired: + // Account lacks the pricing-plan abilities for this endpoint. PagerDuty + // must have authenticated the token before checking plan features, so + // the credential is live. + return true, nil + case http.StatusNotFound: + // Resource not found. Unusual on the /users collection endpoint. + // Not transient -- retrying won't make the endpoint appear. + // Shouldn't happen in practice. return false, nil default: + // Unknown status codes (including 5xx server errors) are potentially + // transient, so return an error to keep the result indeterminate. return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } } diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go index bb57ddf6c31e..58ca0d6aff17 100644 --- a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go +++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go @@ -73,7 +73,7 @@ func TestPagerDutyApiKey_FromChunk(t *testing.T) { wantVerificationErr: true, }, { - name: "found, verified but unexpected api surface", + name: "found, unverified due to not found (404)", s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, args: args{ ctx: context.Background(), @@ -86,9 +86,73 @@ func TestPagerDutyApiKey_FromChunk(t *testing.T) { Verified: false, }, }, + wantErr: false, + }, + { + name: "found, verified via 403 (valid key, insufficient permissions)", + s: Scanner{client: common.ConstantResponseHttpClient(403, `{"error":{"message":"Access Denied","code":2010}}`)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PagerDutyApiKey, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, indeterminate due to rate limiting (429)", + s: Scanner{client: common.ConstantResponseHttpClient(429, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PagerDutyApiKey, + Verified: false, + }, + }, wantErr: false, wantVerificationErr: true, }, + { + name: "found, unverified due to bad request (400)", + s: Scanner{client: common.ConstantResponseHttpClient(400, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PagerDutyApiKey, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "found, verified via 402 (valid key, account plan limitation)", + s: Scanner{client: common.ConstantResponseHttpClient(402, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PagerDutyApiKey, + Verified: true, + }, + }, + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, diff --git a/pkg/detectors/pandadoc/pandadoc.go b/pkg/detectors/pandadoc/pandadoc.go index 9ee3ef07cdb4..9c471a33f546 100644 --- a/pkg/detectors/pandadoc/pandadoc.go +++ b/pkg/detectors/pandadoc/pandadoc.go @@ -3,10 +3,11 @@ package pandadoc import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("API-Key %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pandascore/pandascore.go b/pkg/detectors/pandascore/pandascore.go index e4b24cfa49b8..4caffed79a1c 100644 --- a/pkg/detectors/pandascore/pandascore.go +++ b/pkg/detectors/pandascore/pandascore.go @@ -3,10 +3,11 @@ package pandascore import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else { diff --git a/pkg/detectors/paperform/paperform.go b/pkg/detectors/paperform/paperform.go index 84e78f9e112b..61bd259f310e 100644 --- a/pkg/detectors/paperform/paperform.go +++ b/pkg/detectors/paperform/paperform.go @@ -3,10 +3,11 @@ package paperform import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/paralleldots/paralleldots.go b/pkg/detectors/paralleldots/paralleldots.go index fd65488c44b7..37d900ce8fb6 100644 --- a/pkg/detectors/paralleldots/paralleldots.go +++ b/pkg/detectors/paralleldots/paralleldots.go @@ -3,12 +3,13 @@ package paralleldots import ( "bytes" "context" - regexp "github.com/wasilibs/go-re2" "io" "mime/multipart" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -48,42 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := &bytes.Buffer{} - writer := multipart.NewWriter(payload) - fw, err := writer.CreateFormField("api_key") - if err != nil { - continue - } - _, err = io.Copy(fw, strings.NewReader(resMatch)) - if err != nil { - continue - } - fw, err = writer.CreateFormField("text") - if err != nil { - continue - } - _, err = io.Copy(fw, strings.NewReader("sample text")) - if err != nil { - continue - } - writer.Close() - req, err := http.NewRequestWithContext(ctx, "POST", "https://apis.paralleldots.com/v4/intent", bytes.NewReader(payload.Bytes())) - if err != nil { - continue - } - req.Header.Add("Content-Type", writer.FormDataContentType()) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -92,6 +60,48 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + payload := &bytes.Buffer{} + writer := multipart.NewWriter(payload) + fw, err := writer.CreateFormField("api_key") + if err != nil { + return false, err + } + _, err = io.Copy(fw, strings.NewReader(resMatch)) + if err != nil { + return false, err + } + fw, err = writer.CreateFormField("text") + if err != nil { + return false, err + } + _, err = io.Copy(fw, strings.NewReader("sample text")) + if err != nil { + return false, err + } + _ = writer.Close() + req, err := http.NewRequestWithContext(ctx, "POST", "https://apis.paralleldots.com/v4/intent", bytes.NewReader(payload.Bytes())) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", writer.FormDataContentType()) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_ParallelDots } diff --git a/pkg/detectors/parsehub/parsehub.go b/pkg/detectors/parsehub/parsehub.go index 0323924561d3..1732a52c5c41 100644 --- a/pkg/detectors/parsehub/parsehub.go +++ b/pkg/detectors/parsehub/parsehub.go @@ -3,10 +3,11 @@ package parsehub import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/parsers/parsers.go b/pkg/detectors/parsers/parsers.go index f71d32e3234c..1398fecff741 100644 --- a/pkg/detectors/parsers/parsers.go +++ b/pkg/detectors/parsers/parsers.go @@ -2,10 +2,11 @@ package parsers import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/parseur/parseur.go b/pkg/detectors/parseur/parseur.go index 0d068f1333c5..edf5a04a7005 100644 --- a/pkg/detectors/parseur/parseur.go +++ b/pkg/detectors/parseur/parseur.go @@ -73,7 +73,7 @@ func verifyResult(ctx context.Context, client *http.Client, token string) (bool, return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { return true, nil } diff --git a/pkg/detectors/partnerstack/partnerstack.go b/pkg/detectors/partnerstack/partnerstack.go index 3ab42b13cd84..22dd40178339 100644 --- a/pkg/detectors/partnerstack/partnerstack.go +++ b/pkg/detectors/partnerstack/partnerstack.go @@ -3,10 +3,11 @@ package partnerstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pastebin/pastebin.go b/pkg/detectors/pastebin/pastebin.go index 5b585fb42576..c5d6d3f36af4 100644 --- a/pkg/detectors/pastebin/pastebin.go +++ b/pkg/detectors/pastebin/pastebin.go @@ -3,12 +3,13 @@ package pastebin import ( "bytes" "context" - regexp "github.com/wasilibs/go-re2" "io" "mime/multipart" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -74,7 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - writer.Close() + _ = writer.Close() req, err := http.NewRequestWithContext(ctx, "POST", "https://pastebin.com/api/api_post.php", bytes.NewReader(body.Bytes())) if err != nil { continue @@ -82,7 +83,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", writer.FormDataContentType()) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/paydirtapp/paydirtapp.go b/pkg/detectors/paydirtapp/paydirtapp.go index 82fe2422a067..6e2560e8574e 100644 --- a/pkg/detectors/paydirtapp/paydirtapp.go +++ b/pkg/detectors/paydirtapp/paydirtapp.go @@ -2,10 +2,11 @@ package paydirtapp import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/paymoapp/paymoapp.go b/pkg/detectors/paymoapp/paymoapp.go index 58940fc00f3f..5eda513d2dbb 100644 --- a/pkg/detectors/paymoapp/paymoapp.go +++ b/pkg/detectors/paymoapp/paymoapp.go @@ -3,10 +3,11 @@ package paymoapp import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/paymongo/paymongo.go b/pkg/detectors/paymongo/paymongo.go index 20b0a2d61173..4f97ce5b66ac 100644 --- a/pkg/detectors/paymongo/paymongo.go +++ b/pkg/detectors/paymongo/paymongo.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, "") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/paypaloauth/paypaloauth.go b/pkg/detectors/paypaloauth/paypaloauth.go index c9f15264e770..6cd467d8ef51 100644 --- a/pkg/detectors/paypaloauth/paypaloauth.go +++ b/pkg/detectors/paypaloauth/paypaloauth.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -66,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", encoded)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/paystack/paystack.go b/pkg/detectors/paystack/paystack.go index 5326f0919b04..c7294da3e64e 100644 --- a/pkg/detectors/paystack/paystack.go +++ b/pkg/detectors/paystack/paystack.go @@ -3,10 +3,11 @@ package paystack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pdflayer/pdflayer.go b/pkg/detectors/pdflayer/pdflayer.go index 24b400f1871b..710ce447ab8a 100644 --- a/pkg/detectors/pdflayer/pdflayer.go +++ b/pkg/detectors/pdflayer/pdflayer.go @@ -3,11 +3,12 @@ package pdflayer import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `Contents`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/pdfshift/pdfshift.go b/pkg/detectors/pdfshift/pdfshift.go index 8cb9d0a6dce8..a89545cbb504 100644 --- a/pkg/detectors/pdfshift/pdfshift.go +++ b/pkg/detectors/pdfshift/pdfshift.go @@ -2,10 +2,11 @@ package pdfshift import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth("api", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/peopledatalabs/peopledatalabs.go b/pkg/detectors/peopledatalabs/peopledatalabs.go index 951e5a1775e9..e13119861843 100644 --- a/pkg/detectors/peopledatalabs/peopledatalabs.go +++ b/pkg/detectors/peopledatalabs/peopledatalabs.go @@ -2,10 +2,11 @@ package peopledatalabs import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pepipost/pepipost.go b/pkg/detectors/pepipost/pepipost.go index 18be9866f439..fa2029e75547 100644 --- a/pkg/detectors/pepipost/pepipost.go +++ b/pkg/detectors/pepipost/pepipost.go @@ -2,10 +2,11 @@ package pepipost import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("api_key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/percy/percy.go b/pkg/detectors/percy/percy.go index 81c45344c7fc..619beddef8d2 100644 --- a/pkg/detectors/percy/percy.go +++ b/pkg/detectors/percy/percy.go @@ -3,10 +3,11 @@ package percy import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pganalyzereadkey/pganalyzereadkey.go b/pkg/detectors/pganalyzereadkey/pganalyzereadkey.go new file mode 100644 index 000000000000..c12fa4a21661 --- /dev/null +++ b/pkg/detectors/pganalyzereadkey/pganalyzereadkey.go @@ -0,0 +1,144 @@ +package pganalyzereadkey + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // pganalyze Read API keys use the format: + // pgar_<27 alphanumeric characters> + // + // Example: + // pgar_abcdefghijklmnopqrstuvwxyz12 + pganalyzeTokenPat = regexp.MustCompile( + `\b(pgar_[A-Za-z0-9]{27})\b`, + ) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{ + "pgar_", + } +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for pganalyze API tokens and optionally verifies them. +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + + matches := pganalyzeTokenPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + uniqueTokens[match[1]] = struct{}{} + } + + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_PgAnalyzeReadKey, + Raw: []byte(token), + SecretParts: map[string]string{ + "key": token, + "access_type": "read", + }, + } + + if verify { + verified, verificationErr := verifyPganalyzeToken( + ctx, + s.getClient(), + token, + ) + + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + + return +} + +func verifyPganalyzeToken( + ctx context.Context, + client *http.Client, + token string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + "https://app.pganalyze.com/graphql", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Token "+token) + + res, err := client.Do(req) + if err != nil { + return false, err + } + + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + + case http.StatusOK: + return true, nil + + // Explicit invalid auth + case http.StatusUnauthorized: + return false, nil + + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_PgAnalyzeReadKey +} + +func (s Scanner) Description() string { + return "pganalyze is a PostgreSQL monitoring and performance analysis platform. pganalyze Read API keys can be used to access read-only monitoring and query performance data." +} diff --git a/pkg/detectors/pganalyzereadkey/pganalyzereadkey_integration_test.go b/pkg/detectors/pganalyzereadkey/pganalyzereadkey_integration_test.go new file mode 100644 index 000000000000..ce52063ec463 --- /dev/null +++ b/pkg/detectors/pganalyzereadkey/pganalyzereadkey_integration_test.go @@ -0,0 +1,215 @@ +//go:build detectors +// +build detectors + +package pganalyzereadkey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestPgAnalyzeReadKey_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + testSecrets, err := common.GetSecret( + ctx, + "trufflehog-testing", + "detectors6", + ) + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeToken := testSecrets.MustGetField("PGANALYZE_READ_KEY") + inactiveToken := "pgar_abcdefghijklmnopqrstuvwxyz1" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using pganalyze read key %s for dashboard access", + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PgAnalyzeReadKey, + Verified: true, + Raw: []byte(activeToken), + }, + }, + }, + { + name: "found, real token, verification error due to timeout", + s: Scanner{ + client: common.SaneHttpClientTimeOut(1 * time.Microsecond), + }, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using pganalyze read key %s for dashboard access", + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PgAnalyzeReadKey, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, verification error due to unexpected api surface", + s: Scanner{ + client: common.ConstantResponseHttpClient(500, "{}"), + }, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using pganalyze read key %s for dashboard access", + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PgAnalyzeReadKey, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "Using pganalyze read key %s for dashboard access", + inactiveToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_PgAnalyzeReadKey, + Verified: false, + Raw: []byte(inactiveToken), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData( + tt.args.ctx, + tt.args.verify, + tt.args.data, + ) + + if (err != nil) != tt.wantErr { + t.Fatalf( + "PgAnalyzeReadKey.FromData() error = %v, wantErr %v", + err, + tt.wantErr, + ) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf( + "PgAnalyzeReadKey.FromData() %s diff: (-got +want)\n%s", + tt.name, + diff, + ) + } + }) + } +} + +func BenchmarkPgAnalyzeReadKey_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/pganalyzereadkey/pganalyzereadkey_test.go b/pkg/detectors/pganalyzereadkey/pganalyzereadkey_test.go new file mode 100644 index 000000000000..cad109342f33 --- /dev/null +++ b/pkg/detectors/pganalyzereadkey/pganalyzereadkey_test.go @@ -0,0 +1,138 @@ +package pganalyzereadkey + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestPgAnalyzeReadKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - basic", + input: ` + [INFO] pganalyze initialized + [DEBUG] token=pgar_abcdefghijklmnopqrstuvwxyz1 + [INFO] done + `, + want: []string{ + "pgar_abcdefghijklmnopqrstuvwxyz1", + }, + }, + { + name: "valid pattern - env variable", + input: ` + PGANALYZE_READ_KEY=pgar_123456789012345678901234567 + `, + want: []string{ + "pgar_123456789012345678901234567", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + pgar_aaaaaaaaaaaaaaaaaaaaaaaaaaa + pgar_bbbbbbbbbbbbbbbbbbbbbbbbbbb + `, + want: []string{ + "pgar_aaaaaaaaaaaaaaaaaaaaaaaaaaa", + "pgar_bbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + }, + { + name: "invalid pattern - uppercase prefix", + input: ` + PGAR_abcdefghijklmnopqrstuvwxyz1 + `, + want: nil, + }, + { + name: "invalid pattern - too short", + input: ` + pgar_12345 + `, + want: nil, + }, + { + name: "invalid pattern - too long", + input: ` + pgar_abcdefghijklmnopqrstuvwxyz123456 + `, + want: nil, + }, + { + name: "invalid pattern - invalid characters", + input: ` + pgar_abcde!ghijklmnopqrstuvwxyz12 + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + pgar_ + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData( + context.Background(), + false, + []byte(test.input), + ) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/pinata/pinata.go b/pkg/detectors/pinata/pinata.go index 465dda391ec4..d22292f98651 100644 --- a/pkg/detectors/pinata/pinata.go +++ b/pkg/detectors/pinata/pinata.go @@ -2,11 +2,12 @@ package pinata import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -65,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("pinata_secret_api_key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pinecone/models.go b/pkg/detectors/pinecone/models.go new file mode 100644 index 000000000000..1552d8b93a0e --- /dev/null +++ b/pkg/detectors/pinecone/models.go @@ -0,0 +1,20 @@ +package pinecone + +type listIndexesResponse struct { + Indexes []indexInfo `json:"indexes"` +} + +type indexInfo struct { + Name string `json:"name"` + Host string `json:"host"` + Spec indexSpec `json:"spec"` +} + +type indexSpec struct { + Serverless *serverlessSpec `json:"serverless,omitempty"` +} + +type serverlessSpec struct { + Cloud string `json:"cloud"` + Region string `json:"region"` +} diff --git a/pkg/detectors/pinecone/pinecone.go b/pkg/detectors/pinecone/pinecone.go new file mode 100644 index 000000000000..99e35e6e782d --- /dev/null +++ b/pkg/detectors/pinecone/pinecone.go @@ -0,0 +1,187 @@ +package pinecone + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + "strconv" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // Pinecone API keys follow the pattern: pcsk_{label}_{secret} + // where label is 5-6 alphanumeric chars and secret is exactly 63 alphanumeric + // chars (total length 74-75). Tight bounds prevent over-reading into adjacent + // text and rule out partial/malformed matches. + // Group 1 = whole token, group 2 = label (surfaced as ExtraData["key_id"]). + keyPat = regexp.MustCompile(`\b(pcsk_([A-Za-z0-9]{5,6})_[A-Za-z0-9]{63})\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { + return []string{"pcsk_"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData will find and optionally verify Pinecone secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueMatches := make(map[string]string) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueMatches[match[1]] = match[2] + } + + for token, keyID := range uniqueMatches { + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(token), + Redacted: token[:8] + "..." + token[len(token)-4:], + ExtraData: map[string]string{"key_id": keyID}, + SecretParts: map[string]string{"key": token}, + } + + if verify { + isVerified, extraData, verificationErr := s.verifyMatch(ctx, s.getClient(), token) + s1.Verified = isVerified + maps.Copy(s1.ExtraData, extraData) + s1.SetVerificationError(verificationErr, token) + } + + results = append(results, s1) + } + + return results, nil +} + +// verifyMatch calls the Pinecone List Indexes endpoint (GET /indexes) to validate the key. +// Per the Pinecone API reference, this control-plane endpoint returns 200/401/500. +// 403 is not documented but is observed in practice for valid keys that lack +// ControlPlane permissions (e.g. DataPlaneViewer role). +func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.pinecone.io/indexes", http.NoBody) + if err != nil { + return false, nil, err + } + req.Header.Set("Api-Key", token) + req.Header.Set("X-Pinecone-Api-Version", "2025-10") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return false, nil, err + } + var apiResp listIndexesResponse + if err := json.Unmarshal(bodyBytes, &apiResp); err != nil { + return false, nil, fmt.Errorf("failed to decode 200 response: %w", err) + } + // Guard against a generic 200 that doesn't have the "indexes" key at all: + // JSON unmarshal leaves Indexes as nil in that case, which we treat as a + // malformed response rather than a legitimate empty-indexes result. + if apiResp.Indexes == nil { + return false, nil, fmt.Errorf("unexpected response body structure") + } + return true, buildIndexExtraData(apiResp.Indexes), nil + + case http.StatusUnauthorized: + return false, nil, nil + + case http.StatusForbidden: + // 403 means the key authenticated but lacks permission for this endpoint. + // Official docs do not mention this, but the apikey creation allows setting custom permissions, + // and we have observed 403 responses in practice for valid keys with insufficient permissions. + // We treat this as "verified but with limited permissions" rather than indeterminate + return true, map[string]string{"permission": "restricted"}, nil + + default: + return false, nil, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode) + } +} + +// extractProjectID pulls the project slug from a Pinecone index host string. +// Hosts follow the pattern: {index-name}-{project_id}.svc.{env}.pinecone.io +func extractProjectID(host string) string { + prefix, _, ok := strings.Cut(host, ".svc.") + if !ok { + return "" + } + lastDash := strings.LastIndex(prefix, "-") + if lastDash == -1 { + return "" + } + return prefix[lastDash+1:] +} + +// buildIndexExtraData derives human-useful context (project id, up to 5 index +// summaries) from the list-indexes response. Returns nil when there are no +// indexes to describe. +func buildIndexExtraData(indexes []indexInfo) map[string]string { + extraData := map[string]string{ + "total_indexes": strconv.Itoa(len(indexes)), + } + + for _, idx := range indexes { + if pid := extractProjectID(idx.Host); pid != "" { + extraData["project_id"] = pid + break + } + } + + for i, idx := range indexes { + if i >= 5 { + break + } + prefix := fmt.Sprintf("index_%d_", i) + extraData[prefix+"name"] = idx.Name + extraData[prefix+"host"] = idx.Host + if idx.Spec.Serverless != nil { + extraData[prefix+"cloud"] = idx.Spec.Serverless.Cloud + extraData[prefix+"region"] = idx.Spec.Serverless.Region + } + } + + return extraData +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_Pinecone +} + +func (s Scanner) Description() string { + return "Pinecone is a vector database service. API keys can be used to manage indexes and perform vector operations." +} diff --git a/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey_integration_test.go b/pkg/detectors/pinecone/pinecone_integration_test.go similarity index 71% rename from pkg/detectors/bingsubscriptionkey/bingsubscriptionkey_integration_test.go rename to pkg/detectors/pinecone/pinecone_integration_test.go index fee2aaa7cc7b..c31964898b59 100644 --- a/pkg/detectors/bingsubscriptionkey/bingsubscriptionkey_integration_test.go +++ b/pkg/detectors/pinecone/pinecone_integration_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package bingsubscriptionkey +package pinecone import ( "context" @@ -17,15 +17,15 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -func TestBingsubscriptionkey_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) +func TestPinecone_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("BING_SUBSCRIPTION_KEY") - inactiveSecret := testSecrets.MustGetField("BING_SUBSCRIPTION_KEY_INACTIVE") + secret := testSecrets.MustGetField("PINECONE") + inactiveSecret := testSecrets.MustGetField("PINECONE_INACTIVE") type args struct { ctx context.Context @@ -45,12 +45,12 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bing subscription key %s within", secret)), + data: []byte(fmt.Sprintf("You can find a pinecone secret %s within", secret)), verify: true, }, want: []detectors.Result{ { - DetectorType: detector_typepb.DetectorType_BingSubscriptionKey, + DetectorType: detector_typepb.DetectorType_Pinecone, Verified: true, }, }, @@ -62,12 +62,12 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bing subscription key %s within but not valid", inactiveSecret)), + data: []byte(fmt.Sprintf("You can find a pinecone secret %s within but not valid", inactiveSecret)), verify: true, }, want: []detectors.Result{ { - DetectorType: detector_typepb.DetectorType_BingSubscriptionKey, + DetectorType: detector_typepb.DetectorType_Pinecone, Verified: false, }, }, @@ -79,7 +79,7 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte("You cannot find the key within"), + data: []byte("You cannot find the secret within"), verify: true, }, want: nil, @@ -91,12 +91,12 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bing subscription key %s within", secret)), + data: []byte(fmt.Sprintf("You can find a pinecone secret %s within", secret)), verify: true, }, want: []detectors.Result{ { - DetectorType: detector_typepb.DetectorType_BingSubscriptionKey, + DetectorType: detector_typepb.DetectorType_Pinecone, Verified: false, }, }, @@ -104,16 +104,16 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { wantVerificationErr: true, }, { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + name: "found, unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "")}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bing subscription key %s within", secret)), + data: []byte(fmt.Sprintf("You can find a pinecone secret %s within", secret)), verify: true, }, want: []detectors.Result{ { - DetectorType: detector_typepb.DetectorType_BingSubscriptionKey, + DetectorType: detector_typepb.DetectorType_Pinecone, Verified: false, }, }, @@ -125,7 +125,7 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { - t.Errorf("Bingsubscriptionkey.FromData() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("Pinecone.FromData() error = %v, wantErr %v", err, tt.wantErr) return } for i := range got { @@ -136,9 +136,9 @@ func TestBingsubscriptionkey_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "Redacted", "ExtraData", "SecretParts", "verificationError", "primarySecret") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Bingsubscriptionkey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + t.Errorf("Pinecone.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) } diff --git a/pkg/detectors/pinecone/pinecone_test.go b/pkg/detectors/pinecone/pinecone_test.go new file mode 100644 index 000000000000..4edc1b7cb404 --- /dev/null +++ b/pkg/detectors/pinecone/pinecone_test.go @@ -0,0 +1,304 @@ +package pinecone + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + // 6-char label, 63-char secret (75 chars total) + validKeyLong = "pcsk_T5Afk6_5qU9s3iLVFmaSaJtMat7gTHaT9fXa7ykiBk7iz4uUMuLGLemkdutTgwJevYhqtn" + // 5-char label, 63-char secret (74 chars total) + validKeyShort = "pcsk_wtQV4_J9qqVGjiMW81LJ9H59iajZMWMedenyLnGVR3vbCWq6V3oaEvcQYwPFQpupFUth1" + invalidKeyNoID = "pcsk__5qU9s3iLVFmaSaJtMat7gTHaT9fXa7ykiBk7iz4uUMuLGLemkdutTgwJevYhqtn" + invalidPrefix = "pineconeT5Afk6_5qU9s3iLVFmaSaJtMat7gTHaT9fXa7ykiBk7iz4uUMuLGLemkdutTgwJevYhqtn" + keyword = "pinecone" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func TestPinecone_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - env var assignment", + input: keyword + "_API_KEY=" + validKeyLong, + want: []string{validKeyLong}, + }, + { + name: "valid pattern - short label", + input: "export PINECONE_API_KEY=" + validKeyShort, + want: []string{validKeyShort}, + }, + { + name: "valid pattern - config file", + input: ` + pinecone: + api_key: "` + validKeyLong + `" + environment: us-east-1-aws + `, + want: []string{validKeyLong}, + }, + { + name: "valid pattern - multiple distinct keys", + input: "primary=" + validKeyLong + " secondary=" + validKeyShort, + want: []string{validKeyLong, validKeyShort}, + }, + { + name: "invalid pattern - missing key_id", + input: invalidKeyNoID, + want: []string{}, + }, + { + name: "invalid pattern - wrong prefix", + input: invalidPrefix, + want: []string{}, + }, + { + name: "invalid pattern - secret too short", + input: "pcsk_abcd1_tooShort", + want: []string{}, + }, + { + name: "invalid pattern - label too short (4 chars)", + input: "pcsk_abcd_5qU9s3iLVFmaSaJtMat7gTHaT9fXa7ykiBk7iz4uUMuLGLemkdutTgwJevYhqtn", + want: []string{}, + }, + { + name: "invalid pattern - secret has trailing alphanumeric", + input: validKeyLong + "EXTRA", + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 && len(test.want) > 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + t.Errorf("expected %d results, got %d", len(test.want), len(results)) + for _, r := range results { + t.Logf("got: %s", string(r.Raw)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + actual[string(r.Raw)] = struct{}{} + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + +func TestPinecone_ExtractProjectID(t *testing.T) { + tests := []struct { + name string + host string + want string + }{ + { + name: "serverless host", + host: "my-index-abc1234.svc.us-east1-aws.pinecone.io", + want: "abc1234", + }, + { + name: "hyphenated index name", + host: "my-cool-index-xyz9876.svc.us-west-2.pinecone.io", + want: "xyz9876", + }, + { + name: "missing .svc. segment", + host: "my-index.pinecone.io", + want: "", + }, + { + name: "missing hyphen before .svc.", + host: "abc.svc.us-east1.pinecone.io", + want: "", + }, + { + name: "empty host", + host: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractProjectID(tt.host) + if got != tt.want { + t.Errorf("extractProjectID(%q) = %q, want %q", tt.host, got, tt.want) + } + }) + } +} + +func TestPinecone_VerifyMatchSuccess(t *testing.T) { + scanner := Scanner{client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + t.Fatalf("expected GET request, got %s", req.Method) + } + if req.URL.String() != "https://api.pinecone.io/indexes" { + t.Fatalf("unexpected URL %s", req.URL.String()) + } + if got := req.Header.Get("Api-Key"); got != validKeyLong { + t.Fatalf("unexpected api key header %q", got) + } + if got := req.Header.Get("X-Pinecone-Api-Version"); got != "2025-10" { + t.Fatalf("unexpected api version header %q", got) + } + + body := `{"indexes":[{"name":"example-index","host":"example-index-abc1234.svc.us-east1-aws.pinecone.io","spec":{"serverless":{"cloud":"aws","region":"us-east-1"}}}]}` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + Header: make(http.Header), + Request: req, + }, nil + }), + }} + + verified, extraData, err := scanner.verifyMatch(context.Background(), scanner.client, validKeyLong) + if err != nil { + t.Fatalf("verifyMatch returned error: %v", err) + } + if !verified { + t.Fatal("expected token to verify successfully") + } + if extraData["total_indexes"] != "1" { + t.Fatalf("expected total_indexes=1, got %q", extraData["total_indexes"]) + } + if extraData["project_id"] != "abc1234" { + t.Fatalf("expected project_id=abc1234, got %q", extraData["project_id"]) + } + if extraData["index_0_name"] != "example-index" { + t.Fatalf("expected index_0_name to be populated, got %q", extraData["index_0_name"]) + } +} + +func TestPinecone_VerifyMatchRejectsMissingIndexesKey(t *testing.T) { + scanner := Scanner{client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"projects":[]}`)), + Header: make(http.Header), + Request: req, + }, nil + }), + }} + + verified, extraData, err := scanner.verifyMatch(context.Background(), scanner.client, validKeyLong) + if err == nil { + t.Fatal("expected an error for malformed 200 response") + } + if !strings.Contains(err.Error(), "unexpected response body structure") { + t.Fatalf("unexpected error: %v", err) + } + if verified { + t.Fatal("expected malformed 200 response to remain unverified") + } + if extraData != nil { + t.Fatalf("expected no extra data, got %#v", extraData) + } +} + +func TestPinecone_VerifyMatchRejectsInvalidIndexesPayload(t *testing.T) { + scanner := Scanner{client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"indexes":{}}`)), + Header: make(http.Header), + Request: req, + }, nil + }), + }} + + verified, extraData, err := scanner.verifyMatch(context.Background(), scanner.client, validKeyLong) + if err == nil { + t.Fatal("expected a decode error for invalid indexes payload") + } + if !strings.Contains(err.Error(), "failed to decode 200 response") { + t.Fatalf("unexpected error: %v", err) + } + if verified { + t.Fatal("expected invalid indexes payload to remain unverified") + } + if extraData != nil { + t.Fatalf("expected no extra data, got %#v", extraData) + } +} + +func TestPinecone_FromDataPreservesMetadataOnVerificationError(t *testing.T) { + scanner := Scanner{ + client: &http.Client{ + Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"indexes":{}}`)), + Header: make(http.Header), + Request: req, + }, nil + }), + }, + } + + results, err := scanner.FromData(context.Background(), true, []byte(validKeyLong)) + if err != nil { + t.Fatalf("FromData returned error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected one result, got %d", len(results)) + } + if results[0].Verified { + t.Fatal("expected malformed verification response to keep result unverified") + } + if results[0].VerificationError() == nil { + t.Fatal("expected malformed verification response to set a verification error") + } + if got := results[0].ExtraData["key_id"]; got != "T5Afk6" { + t.Fatalf("expected key_id=T5Afk6, got %q", got) + } + if got := results[0].SecretParts["key"]; got != validKeyLong { + t.Fatalf("expected secret key part to be preserved, got %q", got) + } +} diff --git a/pkg/detectors/pipedream/pipedream.go b/pkg/detectors/pipedream/pipedream.go index ad48747376e4..2048e2d4108d 100644 --- a/pkg/detectors/pipedream/pipedream.go +++ b/pkg/detectors/pipedream/pipedream.go @@ -3,11 +3,12 @@ package pipedream import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pipedrive/pipedrive.go b/pkg/detectors/pipedrive/pipedrive.go index 587de4427e96..81e00984887f 100644 --- a/pkg/detectors/pipedrive/pipedrive.go +++ b/pkg/detectors/pipedrive/pipedrive.go @@ -3,10 +3,11 @@ package pipedrive import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pivotaltracker/pivotaltracker.go b/pkg/detectors/pivotaltracker/pivotaltracker.go index c872968bcaaf..23aa0e837a2d 100644 --- a/pkg/detectors/pivotaltracker/pivotaltracker.go +++ b/pkg/detectors/pivotaltracker/pivotaltracker.go @@ -2,9 +2,10 @@ package pivotaltracker import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-TrackerToken", token) res, err := client.Do(req) if err == nil { - res.Body.Close() // The request body is unused. + _ = res.Body.Close() // The request body is unused. if res.StatusCode >= 200 && res.StatusCode < 300 { result.Verified = true diff --git a/pkg/detectors/pixabay/pixabay.go b/pkg/detectors/pixabay/pixabay.go index 37da38040b0c..efdae36956d9 100644 --- a/pkg/detectors/pixabay/pixabay.go +++ b/pkg/detectors/pixabay/pixabay.go @@ -3,10 +3,11 @@ package pixabay import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/plaidkey/plaidkey.go b/pkg/detectors/plaidkey/plaidkey.go index ff5e5f96d9f2..74a8ac899a5e 100644 --- a/pkg/detectors/plaidkey/plaidkey.go +++ b/pkg/detectors/plaidkey/plaidkey.go @@ -76,6 +76,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_PlaidKey, Raw: []byte(secret), RawV2: []byte(fmt.Sprintf(`%s:%s:%s`, secret, id, token)), + SecretParts: map[string]string{ + "secret": secret, + "id": id, + "token": token, + }, } if verify { @@ -87,13 +92,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", environment)} s1.SetVerificationError(verificationErr, id, secret) - if s1.Verified { - s1.SecretParts = map[string]string{ - "secret": secret, - "id": id, - "token": token, - } - } } results = append(results, s1) } @@ -116,7 +114,7 @@ func verifyMatch(ctx context.Context, client *http.Client, id string, secret str if err != nil { return false, nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/planetscale/planetscale.go b/pkg/detectors/planetscale/planetscale.go index bc50e00f16b7..6b47107a382c 100644 --- a/pkg/detectors/planetscale/planetscale.go +++ b/pkg/detectors/planetscale/planetscale.go @@ -45,6 +45,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_PlanetScale, Raw: []byte(credentials), + SecretParts: map[string]string{ + "id": username, + "token": password, + }, } if verify { @@ -64,13 +68,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // Send HTTP request res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true - s1.SecretParts = map[string]string{ - "id": username, - "token": password, - } } else if res.StatusCode == 401 { // The secret is determinately not verified s1.Verified = false diff --git a/pkg/detectors/planetscaledb/planetscaledb.go b/pkg/detectors/planetscaledb/planetscaledb.go index 4a9db7b9a157..8c78de0b5f16 100644 --- a/pkg/detectors/planetscaledb/planetscaledb.go +++ b/pkg/detectors/planetscaledb/planetscaledb.go @@ -70,7 +70,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } else { s1.SetVerificationError(err, password[0]) } - db.Close() + _ = db.Close() } } diff --git a/pkg/detectors/planviewleankit/planviewleankit.go b/pkg/detectors/planviewleankit/planviewleankit.go index 1aa14091d875..185037caa12a 100644 --- a/pkg/detectors/planviewleankit/planviewleankit.go +++ b/pkg/detectors/planviewleankit/planviewleankit.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/planyo/planyo.go b/pkg/detectors/planyo/planyo.go index f85b7ecf193c..b57f295c09a3 100644 --- a/pkg/detectors/planyo/planyo.go +++ b/pkg/detectors/planyo/planyo.go @@ -48,24 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.planyo.com/rest/?method=get_site_info&api_key=%s", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/vnd.planyo+json; version=3") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - validResponse := strings.Contains(body, "data") || strings.Contains(body, "Your Planyo site has expired") - if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -74,6 +59,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://www.planyo.com/rest/?method=get_site_info&api_key=%s", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Accept", "application/vnd.planyo+json; version=3") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + validResponse := strings.Contains(body, "data") || strings.Contains(body, "Your Planyo site has expired") + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Planyo } diff --git a/pkg/detectors/plivo/plivo.go b/pkg/detectors/plivo/plivo.go index c47de4a29406..2af3b51d3543 100644 --- a/pkg/detectors/plivo/plivo.go +++ b/pkg/detectors/plivo/plivo.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -65,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", decodeSecret)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/podio/podio.go b/pkg/detectors/podio/podio.go index 87c9390cb526..a2aef0591b88 100644 --- a/pkg/detectors/podio/podio.go +++ b/pkg/detectors/podio/podio.go @@ -3,10 +3,11 @@ package podio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pollsapi/pollsapi.go b/pkg/detectors/pollsapi/pollsapi.go index 6d0c92319840..34c0abcac3d1 100644 --- a/pkg/detectors/pollsapi/pollsapi.go +++ b/pkg/detectors/pollsapi/pollsapi.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/poloniex/poloniex.go b/pkg/detectors/poloniex/poloniex.go index 584bafb14b32..5215b8df590b 100644 --- a/pkg/detectors/poloniex/poloniex.go +++ b/pkg/detectors/poloniex/poloniex.go @@ -5,13 +5,14 @@ import ( "crypto/hmac" "crypto/sha512" "encoding/hex" - regexp "github.com/wasilibs/go-re2" "net/http" "net/url" "strconv" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -80,7 +81,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Sign", signature) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/polygon/polygon.go b/pkg/detectors/polygon/polygon.go index c206fc309382..5f2794b5bb5b 100644 --- a/pkg/detectors/polygon/polygon.go +++ b/pkg/detectors/polygon/polygon.go @@ -2,10 +2,11 @@ package polygon import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/portainer/portainer.go b/pkg/detectors/portainer/portainer.go index d40a99da183d..11b7ad0739d1 100644 --- a/pkg/detectors/portainer/portainer.go +++ b/pkg/detectors/portainer/portainer.go @@ -75,7 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 || res.StatusCode == 403 { diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.go index 53947379961a..1895d1d695f4 100644 --- a/pkg/detectors/portainertoken/portainertoken.go +++ b/pkg/detectors/portainertoken/portainertoken.go @@ -48,7 +48,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch) if err != nil { - fmt.Printf("\nINVALID URL\n") // if the URL is invalid just move onto the next one continue } @@ -79,7 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/positionstack/positionstack.go b/pkg/detectors/positionstack/positionstack.go index 6e0c3af08fce..6b1695ff07a0 100644 --- a/pkg/detectors/positionstack/positionstack.go +++ b/pkg/detectors/positionstack/positionstack.go @@ -3,10 +3,11 @@ package positionstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/postageapp/postageapp.go b/pkg/detectors/postageapp/postageapp.go index 8f293521554e..79cf4481d9b5 100644 --- a/pkg/detectors/postageapp/postageapp.go +++ b/pkg/detectors/postageapp/postageapp.go @@ -2,10 +2,11 @@ package postageapp import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Transfer-Encoding", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/postbacks/postbacks.go b/pkg/detectors/postbacks/postbacks.go index 341468a80e62..d09d3843f040 100644 --- a/pkg/detectors/postbacks/postbacks.go +++ b/pkg/detectors/postbacks/postbacks.go @@ -2,11 +2,12 @@ package postbacks import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Postbacks-Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go index 7f5fd0db6ad4..e52bac0995ad 100644 --- a/pkg/detectors/postgres/postgres.go +++ b/pkg/detectors/postgres/postgres.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/lib/pq" regexp "github.com/wasilibs/go-re2" @@ -36,6 +38,24 @@ const ( pgDbType = "db_type" ) +// nonConnectionParams are query-string arguments that ORMs append to +// Postgres connection URIs but that are not libpq connection keywords. lib/pq and pgx +// forward any key they don't recognize to the server as a startup runtime parameter, +// which the server rejects with 42704. Exluding them prevents this +var nonConnectionParams = map[string]struct{}{ + "schema": {}, // Prisma: search_path selector + "connection_limit": {}, // Prisma: client-side pool size + "pool_timeout": {}, // Prisma: pool-acquisition wait + "socket_timeout": {}, // Prisma: per-query timeout + "pgbouncer": {}, // Prisma: PgBouncer compatibility mode + "sslidentity": {}, // Prisma: PKCS12 certificate path +} + +func isNonConnectionParam(key string) bool { + _, ok := nonConnectionParams[key] + return ok +} + // This detector currently only finds Postgres connection string URIs // (https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS) When it finds one, it uses // pq.ParseURI to normalize this into space-separated key-value pair Postgres connection string, and then uses a regular @@ -47,9 +67,9 @@ const ( // Multi-host connection string URIs are currently not supported because pq.ParseURI doesn't parse them correctly. If we // happen to run into a case where this matters we can address it then. var ( - _ detectors.Detector = (*Scanner)(nil) - uriPattern = regexp.MustCompile(`\b(?i)(postgres(?:ql)?)://\S+\b`) - connStrPartPattern = regexp.MustCompile(`([[:alpha:]]+)='(.+?)' ?`) + _ detectors.Detector = (*Scanner)(nil) + uriPattern = regexp.MustCompile(`\b(?i)(postgres(?:ql)?)://\S+\b`) + connStrPartPattern = regexp.MustCompile(`([[:alpha:]_]+)='(.+?)' ?`) ) type Scanner struct { @@ -58,6 +78,11 @@ type Scanner struct { ignorePatterns []*regexp.Regexp } +type uriMatch struct { + params map[string]string + rawURI string +} + func New(opts ...func(*Scanner)) *Scanner { scanner := &Scanner{ ignorePatterns: []*regexp.Regexp{}, @@ -93,12 +118,13 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { var results []detectors.Result - candidateParamSets := findUriMatches(data, s.ignorePatterns) + candidateURIs := findUriMatches(data, s.ignorePatterns) - for _, params := range candidateParamSets { + for _, candidateURI := range candidateURIs { if common.IsDone(ctx) { break } + params := candidateURI.params user, ok := params[pgUser] if !ok { continue @@ -139,7 +165,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete DetectorType: detector_typepb.DetectorType_Postgres, Raw: raw, RawV2: raw, + SecretParts: map[string]string{"connection_string": string(raw)}, } + // Set the un-normalized raw match as the primary secret value. + // This ensures that the engine's line-offset and ignore-tag matching logic + // (which searches the source document for the exact string) can locate the match, + // even though Raw/RawV2 are stored in a normalized form. + result.SetPrimarySecretValue(candidateURI.rawURI) // We don't need to normalize the (deprecated) requiressl option into the (up-to-date) sslmode option - pq can // do it for us - but we will do it anyway here so that when we later capture sslmode into ExtraData we will @@ -154,18 +186,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete if verify { // pq appears to ignore the context deadline, so we copy any timeout that's been set into the connection // parameters themselves. - if timeout, ok := getDeadlineInSeconds(ctx); ok && timeout > 0 { - params[pgConnectTimeout] = strconv.Itoa(timeout) - } else if ok && timeout <= 0 { - // Deadline in the context has already exceeded. - break - } - - isVerified, verificationErr := verifyPostgres(params) - result.Verified = isVerified - result.SetVerificationError(verificationErr, password) - result.SecretParts = map[string]string{ - "connection_string": string(raw), + timeout, hasDeadline := getDeadlineInSeconds(ctx) + if hasDeadline && timeout <= 0 { + // The deadline has already passed, so this credential cannot be tested. Report it as indeterminate + // rather than abandoning this candidate and every remaining one. The IsDone check at the top of the + // loop ends the scan on the next iteration. + result.SetVerificationError(context.DeadlineExceeded, password) + } else { + if hasDeadline { + params[pgConnectTimeout] = strconv.Itoa(timeout) + } + + isVerified, verificationErr := verifyPostgres(ctx, params) + result.Verified = isVerified + result.SetVerificationError(verificationErr, password) } } @@ -177,6 +211,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete result.ExtraData = map[string]string{ pgSslmode: sslmode, } + if host != "" { + if port != "" { + result.ExtraData["host"] = host + ":" + port + } else { + result.ExtraData["host"] = host + } + } + if user != "" { + result.ExtraData["username"] = user + } + if dbname := params[pgDbname]; dbname != "" { + result.ExtraData["database"] = dbname + } results = append(results, result) } @@ -188,8 +235,8 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) { return false, "" } -func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []map[string]string { - var matches []map[string]string +func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []uriMatch { + var matches []uriMatch for _, uri := range uriPattern.FindAll(data, -1) { if shouldIgnore(uri, ignorePatterns) { continue @@ -213,7 +260,10 @@ func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []map[string]s } params[pgDbType] = dbType - matches = append(matches, params) + matches = append(matches, uriMatch{ + params: params, + rawURI: string(uri), + }) } return matches } @@ -227,6 +277,13 @@ func shouldIgnore(uri []byte, ignorePatterns []*regexp.Regexp) bool { return false } +// isNeonHost reports whether host is a Neon managed-Postgres endpoint. +// Neon advertises SCRAM-SHA-256 with iteration count i=1; lib/pq rejects that, +// so verification for these hosts uses pgx instead (SCAN-1020). +func isNeonHost(host string) bool { + return strings.HasSuffix(strings.ToLower(host), ".neon.tech") +} + // getDeadlineInSeconds gets the deadline from the context in seconds. If there // is no deadline, false is returned. If the deadline is already exceeded, a // negative or 0 value will be returned. @@ -241,16 +298,97 @@ func getDeadlineInSeconds(ctx context.Context) (int, bool) { return int(duration.Seconds()), true } -func isErrorDatabaseNotFound(err error, dbName string) bool { +// The server looks the database up only after authenticating, so this confirms the credentials. +const invalidCatalogName = "3D000" + +// Message text is only a fallback, for proxies that relay a failure without a SQLSTATE; the server +// translates messages per lc_messages. Postgres substitutes the user name, not "postgres", when a +// connection string names no database (src/backend/tcop/backend_startup.c). +func isErrorDatabaseNotFound(err error, params map[string]string) bool { + var pqErr *pq.Error + if errors.As(err, &pqErr) && pqErr.Code == invalidCatalogName { + return true + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == invalidCatalogName { + return true + } + + dbName := params[pgDbname] if dbName == "" { - dbName = "postgres" + dbName = params[pgUser] + } + if dbName == "" { + return false + } + + return strings.Contains(err.Error(), fmt.Sprintf("database %q does not exist", dbName)) +} + +func verifyPostgres(ctx context.Context, params map[string]string) (bool, error) { + // Neon (managed Postgres) advertises SCRAM-SHA-256 with iteration count i=1. + // lib/pq rejects iteration fields shorter than 6 chars, which traps these + // secrets in indeterminate reverification. pgx accepts any iterations > 0. + if isNeonHost(params[pgHost]) { + return verifyPostgresPgx(ctx, params) } - missingDbErrorText := fmt.Sprintf("database \"%s\" does not exist", dbName) + return verifyPostgresPq(params) +} + +// verifyPostgresPgx verifies credentials with jackc/pgx. Used for Neon hosts +// where lib/pq's SCRAM client cannot complete the handshake (SCAN-1020). +func verifyPostgresPgx(ctx context.Context, params map[string]string) (bool, error) { + conn, err := pgx.Connect(ctx, pgxConnString(params)) + if err != nil { + return classifyPostgresVerifyError(err, params) + } + defer func() { + // Best-effort close after verification; the verify outcome is already decided. + if closeErr := conn.Close(ctx); closeErr != nil { + return + } + }() + + if err := conn.Ping(ctx); err != nil { + return classifyPostgresVerifyError(err, params) + } + return true, nil +} + +// pgxConnString builds a libpq-style connection string for pgx, omitting keys +// that are detector-only (db_type) or libpq client options pgx would forward as +// unrecognized server GUCs (requiressl). sslmode is already normalized in FromData. +func pgxConnString(params map[string]string) string { + var connStr strings.Builder + for key, value := range params { + if key == pgDbType || key == pgRequiressl || isNonConnectionParam(key) { + continue + } + fmt.Fprintf(&connStr, "%s='%s'", key, value) + } + return connStr.String() +} - return strings.Contains(err.Error(), missingDbErrorText) +func classifyPostgresVerifyError(err error, params map[string]string) (bool, error) { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "28P01": // invalid_password + return false, nil + case "3D000": // invalid_catalog_name — authenticated, DB missing + return true, nil + } + } + if strings.Contains(err.Error(), "password authentication failed") { + return false, nil + } + if isErrorDatabaseNotFound(err, params) { + return true, nil + } + return false, err } -func verifyPostgres(params map[string]string) (bool, error) { +func verifyPostgresPq(params map[string]string) (bool, error) { if sslmode := params[pgSslmode]; sslmode == pgSslmodeAllow || sslmode == pgSslmodePrefer { // pq doesn't support 'allow' or 'prefer'. If we find either of them, we'll just ignore it. This will trigger // the same logic that is run if no sslmode is set at all (which mimics 'prefer', which is the default). @@ -273,6 +411,9 @@ func verifyPostgres(params map[string]string) (bool, error) { var connStr string for key, value := range params { + if isNonConnectionParam(key) { + continue + } connStr += fmt.Sprintf("%s='%s'", key, value) } @@ -280,7 +421,7 @@ func verifyPostgres(params map[string]string) (bool, error) { if err != nil { return false, err } - defer db.Close() + defer func() { _ = db.Close() }() err = db.Ping() switch { @@ -294,8 +435,8 @@ func verifyPostgres(params map[string]string) (bool, error) { // connections are acceptable, so now we try a connection without SSL. params[pgSslmode] = pgSslmodeDisable defer delete(params, pgSslmode) // We want to return with the original params map intact (for ExtraData) - return verifyPostgres(params) - case isErrorDatabaseNotFound(err, params[pgDbname]): + return verifyPostgresPq(params) + case isErrorDatabaseNotFound(err, params): return true, nil // If we know this, we were able to authenticate default: return false, err diff --git a/pkg/detectors/postgres/postgres_integration_test.go b/pkg/detectors/postgres/postgres_integration_test.go index d78b5cc7d77a..26cbda1d3113 100644 --- a/pkg/detectors/postgres/postgres_integration_test.go +++ b/pkg/detectors/postgres/postgres_integration_test.go @@ -335,7 +335,7 @@ func TestPostgres_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "SecretParts", "ExtraData") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go index 85f5b82eb507..465682972a78 100644 --- a/pkg/detectors/postgres/postgres_test.go +++ b/pkg/detectors/postgres/postgres_test.go @@ -2,10 +2,15 @@ package postgres import ( "context" + "errors" "fmt" "testing" "github.com/google/go-cmp/cmp" + "github.com/jackc/pgx/v5/pgconn" + "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -82,17 +87,237 @@ func TestPostgres_Pattern(t *testing.T) { } } +func TestPostgres_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + wantDatabase string + }{ + { + name: "standard URI with database", + data: "postgres://myuser:mypass@dbhost.example.com:5432/mydb", + wantHost: "dbhost.example.com:5432", + wantUsername: "myuser", + wantDatabase: "mydb", + }, + { + name: "postgresql scheme", + data: "postgresql://admin:secret@10.0.0.1:5433/production", + wantHost: "10.0.0.1:5433", + wantUsername: "admin", + wantDatabase: "production", + }, + { + name: "without database", + data: "postgres://sN19x:d7N8bs@1.2.3.4:5432?sslmode=require", + wantHost: "1.2.3.4:5432", + wantUsername: "sN19x", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{detectLoopback: true} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + assert.Equal(t, tt.wantDatabase, r.ExtraData["database"]) + assert.Contains(t, r.ExtraData, "sslmode", "ExtraData[sslmode] should still be present") + }) + } +} + +func TestIsNeonHost(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + host string + want bool + }{ + {host: "ep-falling-feather-aimmxil4.c-4.us-east-1.aws.neon.tech", want: true}, + {host: "EP.NEON.TECH", want: true}, + {host: "db.example.com", want: false}, + {host: "neon.tech.evil.com", want: false}, + {host: "neon.tech", want: false}, + {host: "", want: false}, + } { + t.Run(tc.host, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.want, isNeonHost(tc.host)) + }) + } +} + +func TestPgxConnStringOmitsClientOnlyParams(t *testing.T) { + t.Parallel() + + got := pgxConnString(map[string]string{ + pgHost: "ep-example.us-east-1.aws.neon.tech", + pgPort: "5432", + pgUser: "user", + pgPassword: "secret", + pgDbname: "neondb", + pgSslmode: pgSslmodeRequire, + pgDbType: "postgres", + pgRequiressl: "1", + }) + + assert.Contains(t, got, "host='ep-example.us-east-1.aws.neon.tech'") + assert.Contains(t, got, "sslmode='require'") + assert.NotContains(t, got, "db_type=") + assert.NotContains(t, got, "requiressl=") +} + +func TestClassifyPostgresVerifyError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + params map[string]string + wantVerified bool + wantErr bool + }{ + { + name: "invalid password code", + err: &pgconn.PgError{Code: "28P01", Message: "password authentication failed"}, + wantVerified: false, + wantErr: false, + }, + { + name: "missing database code", + err: &pgconn.PgError{Code: "3D000", Message: `database "app" does not exist`}, + params: map[string]string{pgDbname: "app"}, + wantVerified: true, + wantErr: false, + }, + { + name: "missing database code needs no parameters", + err: &pgconn.PgError{Code: "3D000", Message: "la base de données n'existe pas"}, + wantVerified: true, + wantErr: false, + }, + { + name: "missing database code from lib/pq", + err: &pq.Error{Code: "3D000", Message: `database "app" does not exist`}, + wantVerified: true, + wantErr: false, + }, + { + name: "password failure by message", + err: errors.New("password authentication failed for user \"x\""), + wantVerified: false, + wantErr: false, + }, + { + name: "missing database by message names the given database", + err: errors.New(`database "app" does not exist`), + params: map[string]string{pgDbname: "app"}, + wantVerified: true, + wantErr: false, + }, + { + name: "missing database by message names the user when no database is given", + err: errors.New(`database "svc_reports" does not exist`), + params: map[string]string{pgUser: "svc_reports"}, + wantVerified: true, + wantErr: false, + }, + { + name: "missing database by message for a database nobody asked for", + err: errors.New(`database "postgres" does not exist`), + params: map[string]string{pgUser: "svc_reports"}, + wantVerified: false, + wantErr: true, + }, + { + name: "proxy relays the failure without a sqlstate", + err: errors.New(`server login has been failing, cached error: database "svc_reports" does not exist`), + params: map[string]string{pgUser: "svc_reports"}, + wantVerified: true, + wantErr: false, + }, + { + name: "indeterminate error", + err: errors.New("connection refused"), + wantVerified: false, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + verified, err := classifyPostgresVerifyError(tc.err, tc.params) + assert.Equal(t, tc.wantVerified, verified) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + func TestPostgres_FromDataWithIgnorePattern(t *testing.T) { s := New( WithIgnorePattern([]string{ `1\.2\.3\.4`, })) got, err := s.FromData(context.Background(), false, []byte(validUriPattern)) - if err != nil { - t.Errorf("FromData() error = %v", err) - return + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestPostgres_RawVsPrimarySecret(t *testing.T) { + s := Scanner{} + input := "postgres://user:pass@host/dbname" + results, err := s.FromData(context.Background(), false, []byte(input)) + require.NoError(t, err) + require.Len(t, results, 1) + + res := results[0] + expectedRaw := "postgres://user:pass@host:5432" + assert.Equal(t, expectedRaw, string(res.Raw)) + assert.Equal(t, expectedRaw, string(res.RawV2)) + assert.Equal(t, input, res.GetPrimarySecretValue()) +} + +// TestVerifyConnString_FiltersNonConnectionParams verifies that ORM query-string +// arguments which are not libpq connection keywords are dropped from the verification +// connection string. +func TestVerifyConnString_FiltersNonConnectionParams(t *testing.T) { + uri := `postgresql://u:p@h:5432/db?sslmode=require&schema=public&connection_limit=5&pool_timeout=10&socket_timeout=30&pgbouncer=true&sslidentity=/tmp/i.p12&connect_timeout=7` + + matches := findUriMatches([]byte(uri), nil) + require.Len(t, matches, 1) + got := pgxConnString(matches[0].params) + + + for _, key := range []string{pgUser, pgPassword, pgHost, pgPort, pgDbname, pgSslmode, pgConnectTimeout} { + assert.Containsf(t, got, key+"=", "expected connection param %q to be preserved", key) + } + for _, key := range []string{pgDbType, "schema", "connection_limit", "pool_timeout", "socket_timeout", "pgbouncer", "sslidentity"} { + assert.NotContainsf(t, got, key+"=", "expected non-connection param %q to be filtered out", key) + } + for _, mangled := range []string{"limit=", "identity="} { + assert.NotContainsf(t, got, mangled, "found truncated key fragment %q — connStrPartPattern lost the underscore", mangled) + } +} + +func TestIsNonConnectionParam(t *testing.T) { + // ORM arguments that are not libpq keywords. + for _, key := range []string{"schema", "connection_limit", "pool_timeout", "socket_timeout", "pgbouncer", "sslidentity"} { + assert.Truef(t, isNonConnectionParam(key), "%q should be treated as a non-connection param", key) } - if len(got) != 0 { - t.Errorf("expected no results, but got %d", len(got)) + // Real libpq keywords that ORMs also use — must not be filtered. + for _, key := range []string{pgConnectTimeout, pgSslmode, "sslcert", pgUser, pgPassword, pgHost, pgPort, pgDbname} { + assert.Falsef(t, isNonConnectionParam(key), "%q is a real connection parameter and must not be filtered", key) } } diff --git a/pkg/detectors/posthog/posthog.go b/pkg/detectors/posthog/posthog.go index 157d61c5baa8..1f884085f1af 100644 --- a/pkg/detectors/posthog/posthog.go +++ b/pkg/detectors/posthog/posthog.go @@ -21,7 +21,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`) + keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43,48})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -42,6 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_PosthogApp, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -56,22 +57,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true - s1.SecretParts = map[string]string{ - "key": resMatch, - } } else if res.StatusCode == 401 { // Try EU Endpoint only if other one fails. res, err := client.Do(reqEU) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true - s1.SecretParts = map[string]string{ - "key": resMatch, - } } } } diff --git a/pkg/detectors/posthog/posthog_test.go b/pkg/detectors/posthog/posthog_test.go index 923bbc1702eb..006140607daf 100644 --- a/pkg/detectors/posthog/posthog_test.go +++ b/pkg/detectors/posthog/posthog_test.go @@ -13,6 +13,7 @@ import ( var ( validPattern = "phx_C1rP9fnAtEFJvb0IYCFdeQhar2WdwUFBYHJym1F_Zqr" + validPattern48 = "phx_C1rP9fnAtEFJvb0IYCFdeQhar2WdwUFBYHJym1F_ZqrAbcde" invalidPattern = "phx_C1rP9fnAtEFJvb0IYCF?eQhar2WdwUFBYHJym1F_Zqr" keyword = "posthog" ) @@ -30,6 +31,11 @@ func TestAppPosthog_Pattern(t *testing.T) { input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), want: []string{validPattern}, }, + { + name: "valid pattern 48 chars - with keyword posthog", + input: fmt.Sprintf("%s token = '%s'", keyword, validPattern48), + want: []string{validPattern48}, + }, { name: "invalid pattern", input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), diff --git a/pkg/detectors/postman/postman.go b/pkg/detectors/postman/postman.go index b4540d219fb5..3e7a217ef8a8 100644 --- a/pkg/detectors/postman/postman.go +++ b/pkg/detectors/postman/postman.go @@ -47,6 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Postman, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -54,9 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result isVerified, verificationErr := verifyPostman(ctx, client, resMatch) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resMatch) - s1.SecretParts = map[string]string{ - "key": resMatch, - } } results = append(results, s1) @@ -82,7 +80,7 @@ func verifyPostman(ctx context.Context, client *http.Client, token string) (bool if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/postmark/postmark.go b/pkg/detectors/postmark/postmark.go index b3d2b9c09a38..47b47fabcb67 100644 --- a/pkg/detectors/postmark/postmark.go +++ b/pkg/detectors/postmark/postmark.go @@ -44,6 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Postmark, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } if verify { @@ -51,12 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = valid s1.ExtraData = extraData s1.SetVerificationError(err) - - if valid { - s1.SecretParts = map[string]string{ - "key": resMatch, - } - } } results = append(results, s1) diff --git a/pkg/detectors/powrbot/powrbot.go b/pkg/detectors/powrbot/powrbot.go index 9a1a5d8f7517..2884d80ca90d 100644 --- a/pkg/detectors/powrbot/powrbot.go +++ b/pkg/detectors/powrbot/powrbot.go @@ -3,10 +3,11 @@ package powrbot import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("secret-key %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/prefect/prefect.go b/pkg/detectors/prefect/prefect.go index 0d2761eda49d..1aed862b7702 100644 --- a/pkg/detectors/prefect/prefect.go +++ b/pkg/detectors/prefect/prefect.go @@ -3,10 +3,11 @@ package prefect import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/privacy/privacy.go b/pkg/detectors/privacy/privacy.go index da59c9a05c82..208fb86bf322 100644 --- a/pkg/detectors/privacy/privacy.go +++ b/pkg/detectors/privacy/privacy.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Authorization", "api-key "+resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/privatekey/privatekey.go b/pkg/detectors/privatekey/privatekey.go index a345933ae4a1..d7ea58826ad4 100644 --- a/pkg/detectors/privatekey/privatekey.go +++ b/pkg/detectors/privatekey/privatekey.go @@ -69,6 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result Raw: []byte(token), Redacted: token[0:64], ExtraData: make(map[string]string), + SecretParts: map[string]string{"token": token}, } // set not normalized match as primary secret value so it is used to calculate line of code @@ -80,7 +81,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["encrypted"] = "true" parsedKey, passphrase, err = Crack([]byte(token)) if err != nil { + // The key is encrypted and the passphrase could not be + // recovered from the wordlist. It still represents a real + // exposure (offline-crackable, weak/legacy KDFs, passphrase + // often reused or committed nearby), so surface it as an + // unverified finding instead of dropping it silently. s1.SetVerificationError(err, token) + results = append(results, s1) continue } if passphrase != "" { @@ -152,11 +159,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for k, v := range extraData.data { s1.ExtraData[k] = v } - - // enabled th - s1.SecretParts = map[string]string{ - "token": token, - } } else { s1.ExtraData = nil } @@ -226,7 +228,7 @@ func LookupFingerprint(ctx context.Context, publicKeyFingerprintInHex string) (* if err != nil { return nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() results := DriftwoodResult{} err = json.NewDecoder(res.Body).Decode(&results) diff --git a/pkg/detectors/privatekey/privatekey_test.go b/pkg/detectors/privatekey/privatekey_test.go index 661a8fd96f01..c5f5b76aad93 100644 --- a/pkg/detectors/privatekey/privatekey_test.go +++ b/pkg/detectors/privatekey/privatekey_test.go @@ -96,3 +96,44 @@ func TestPrivatekey_Pattern(t *testing.T) { }) } } + +// encryptedUncrackablePattern is an ed25519 key protected by a passphrase that +// is not present in the built-in wordlist (pkg/detectors/privatekey/list.txt), +// so Crack cannot recover it. +var encryptedUncrackablePattern = `-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABDnvQyInG +vB+yh/WYczTGXbAAAAGAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAILUSv84lMSLVN0iP +knFlHoYobo1oohtLNp/ihBaf76QxAAAAoErDjg2HQs1hgYm0vfyrpBxWrcJ1u/LQG4o6cm +GJ3NnZmMmJemrnMXjGHB3zj63AEPxxyb6BWZH5Olb5gYMgWcnOU91cGvyC7aT6C5cOnFb1 +ZtkdxTxqeUOMFS51gwDT04aURSz/caCV9KN4Y2MCHyW+GWxxD7eL0R6KhyT2j0z2BewCw9 +kRN1fq3C2rTb+wTaNz8q9X0R+6JGxfXGhpTJ0= +-----END OPENSSH PRIVATE KEY----- +` + +// TestPrivatekey_EncryptedKeyReported asserts that a passphrase-protected key +// whose passphrase cannot be cracked is still surfaced as an unverified finding +// rather than being dropped silently (issue #5115). +func TestPrivatekey_EncryptedKeyReported(t *testing.T) { + d := Scanner{} + input := fmt.Sprintf("%s = '%s'", keyword, encryptedUncrackablePattern) + + results, err := d.FromData(context.Background(), false, []byte(input)) + if err != nil { + t.Fatalf("FromData error = %v", err) + } + + if len(results) != 1 { + t.Fatalf("expected 1 result for an encrypted key with an uncrackable passphrase, got %d", len(results)) + } + + r := results[0] + if r.Verified { + t.Errorf("expected the finding to be unverified") + } + if r.VerificationError() == nil { + t.Errorf("expected a verification error to be set on the finding") + } + if r.ExtraData["encrypted"] != "true" { + t.Errorf("expected ExtraData[\"encrypted\"] = \"true\", got %q", r.ExtraData["encrypted"]) + } +} diff --git a/pkg/detectors/privatekey/ssh_integration.go b/pkg/detectors/privatekey/ssh_integration.go index 5f0a8cbc5221..913a1acd80e9 100644 --- a/pkg/detectors/privatekey/ssh_integration.go +++ b/pkg/detectors/privatekey/ssh_integration.go @@ -64,13 +64,13 @@ func firstResponseFromSSH(ctx context.Context, parsedKey any, username, hostport } return "", err } - defer client.Close() + defer func() { _ = client.Close() }() session, err := client.NewSession() if err != nil { return "", err } - defer session.Close() + defer func() { _ = session.Close() }() var output bytes.Buffer session.Stderr = &output @@ -93,7 +93,7 @@ func sshDialWithContext(ctx context.Context, network, addr string, config *ssh.C ncc, chans, reqs, err := ssh.NewClientConn(conn, addr, config) if err != nil { - conn.Close() + _ = conn.Close() return nil, fmt.Errorf("error creating SSH connection to %s: %w", addr, err) } diff --git a/pkg/detectors/prodpad/prodpad.go b/pkg/detectors/prodpad/prodpad.go index f3d7f6726a60..b0417a961bce 100644 --- a/pkg/detectors/prodpad/prodpad.go +++ b/pkg/detectors/prodpad/prodpad.go @@ -3,10 +3,11 @@ package prodpad import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/prospectcrm/prospectcrm.go b/pkg/detectors/prospectcrm/prospectcrm.go index 3ebbbb5334ca..0f26e8c2de3c 100644 --- a/pkg/detectors/prospectcrm/prospectcrm.go +++ b/pkg/detectors/prospectcrm/prospectcrm.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/protocolsio/protocolsio.go b/pkg/detectors/protocolsio/protocolsio.go index 2e70a028f35a..a2ef1470b55b 100644 --- a/pkg/detectors/protocolsio/protocolsio.go +++ b/pkg/detectors/protocolsio/protocolsio.go @@ -3,10 +3,11 @@ package protocolsio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/proxycrawl/proxycrawl.go b/pkg/detectors/proxycrawl/proxycrawl.go index 842c9f09a40a..0579a2d5a169 100644 --- a/pkg/detectors/proxycrawl/proxycrawl.go +++ b/pkg/detectors/proxycrawl/proxycrawl.go @@ -3,11 +3,12 @@ package proxycrawl import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_ProxyCrawl, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go index 281372b32c8e..7229ffd8e49c 100644 --- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go +++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go @@ -91,10 +91,10 @@ func verifyPubNub(ctx context.Context, client *http.Client, resMatch, ressubMatc return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { return true, nil - } else if !(res.StatusCode == 400 || res.StatusCode == 403) { + } else if res.StatusCode != 400 && res.StatusCode != 403 { // 403 is suggested by the API docs (https://www.pubnub.com/docs/sdks/rest-api/send-signal-to-channel) // 400 is what actually seems to be coming back for invalid credentials return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) diff --git a/pkg/detectors/pulumi/pulumi.go b/pkg/detectors/pulumi/pulumi.go index c727abe1a536..cbe9133e0b9c 100644 --- a/pkg/detectors/pulumi/pulumi.go +++ b/pkg/detectors/pulumi/pulumi.go @@ -3,10 +3,11 @@ package pulumi import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/purestake/purestake.go b/pkg/detectors/purestake/purestake.go index e6cc2d5e37d3..8f785568712b 100644 --- a/pkg/detectors/purestake/purestake.go +++ b/pkg/detectors/purestake/purestake.go @@ -2,10 +2,11 @@ package purestake import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey.go b/pkg/detectors/pushbulletapikey/pushbulletapikey.go index f94e8fb1a868..d365d4674153 100644 --- a/pkg/detectors/pushbulletapikey/pushbulletapikey.go +++ b/pkg/detectors/pushbulletapikey/pushbulletapikey.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Access-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/pypi/pypi.go b/pkg/detectors/pypi/pypi.go index 7982041fe65f..cfdc470d899d 100644 --- a/pkg/detectors/pypi/pypi.go +++ b/pkg/detectors/pypi/pypi.go @@ -77,10 +77,11 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, _ = writer.WriteField(":action", "file_upload") _ = writer.WriteField("name", "dummy-package") _ = writer.WriteField("version", "0.0.1") + _ = writer.WriteField("filetype", "sdist") _ = writer.WriteField("content", "dummy-content") // Close the writer to finalize the form - writer.Close() + _ = writer.Close() // Create a new POST request to the PyPI legacy upload URL req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://upload.pypi.org/legacy/", &body) @@ -104,7 +105,8 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, }() // Check for expected status codes for verification - if res.StatusCode == http.StatusBadRequest { + switch res.StatusCode { + case http.StatusBadRequest: verified, err := common.ResponseContainsSubstring(res.Body, "Include at least one message digest.") if err != nil { return false, nil, err @@ -112,7 +114,7 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, if verified { return true, nil, nil } - } else if res.StatusCode == http.StatusForbidden { + case http.StatusForbidden: // If we get a 403 status, the key is invalid return false, nil, nil } diff --git a/pkg/detectors/rabbitmq/rabbitmq.go b/pkg/detectors/rabbitmq/rabbitmq.go index af7fcffb2033..d4ee7b98e774 100644 --- a/pkg/detectors/rabbitmq/rabbitmq.go +++ b/pkg/detectors/rabbitmq/rabbitmq.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_RabbitMQ, Raw: []byte(urlMatch), SecretParts: map[string]string{"key": urlMatch}, - Redacted: strings.TrimSpace(strings.Replace(parsedURL.String(), password, "********", -1)), + Redacted: strings.TrimSpace(strings.ReplaceAll(parsedURL.String(), password, "********")), } if verify { diff --git a/pkg/detectors/railwayapp/railwayapp.go b/pkg/detectors/railwayapp/railwayapp.go index d0f61bca4505..b964c8b07479 100644 --- a/pkg/detectors/railwayapp/railwayapp.go +++ b/pkg/detectors/railwayapp/railwayapp.go @@ -119,7 +119,7 @@ func verifyRailwayApp(ctx context.Context, client *http.Client, match string) (b if err != nil { return false, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() /* GraphQL queries return response with 200 OK status code even for errors diff --git a/pkg/detectors/ramp/ramp.go b/pkg/detectors/ramp/ramp.go index a4d6a3df6df5..b9599d468dba 100644 --- a/pkg/detectors/ramp/ramp.go +++ b/pkg/detectors/ramp/ramp.go @@ -78,7 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/rapidapi/rapidapi.go b/pkg/detectors/rapidapi/rapidapi.go index 5f9bfdcd43a6..f3378cecee06 100644 --- a/pkg/detectors/rapidapi/rapidapi.go +++ b/pkg/detectors/rapidapi/rapidapi.go @@ -2,10 +2,11 @@ package rapidapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-rapidapi-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/raven/raven.go b/pkg/detectors/raven/raven.go index 4e05da24663b..bc741ec39e43 100644 --- a/pkg/detectors/raven/raven.go +++ b/pkg/detectors/raven/raven.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -48,25 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.raventools.com/api?key=%s&method=profile_info&format=json", resMatch), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if json.Valid(bodyBytes) { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -75,6 +60,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.raventools.com/api?key=%s&method=profile_info&format=json", resMatch), nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + if res.StatusCode >= 200 && res.StatusCode < 300 { + if json.Valid(bodyBytes) { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Raven } diff --git a/pkg/detectors/rawg/rawg.go b/pkg/detectors/rawg/rawg.go index c2f89615d64f..9b63105a4091 100644 --- a/pkg/detectors/rawg/rawg.go +++ b/pkg/detectors/rawg/rawg.go @@ -2,10 +2,11 @@ package rawg import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.go index 10cdf4849cef..7263fb3f69a6 100644 --- a/pkg/detectors/razorpay/razorpay.go +++ b/pkg/detectors/razorpay/razorpay.go @@ -57,24 +57,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.razorpay.com/v1/items?count=1", nil) - if err != nil { - continue - } - req.SetBasicAuth(key, secret) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if json.Valid(bodyBytes) { - s1.Verified = true - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, key, secret) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, key, secret) } results = append(results, s1) @@ -85,6 +70,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, key, secret string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.razorpay.com/v1/items?count=1", nil) + if err != nil { + return false, err + } + req.SetBasicAuth(key, secret) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + if res.StatusCode >= 200 && res.StatusCode < 300 && json.Valid(bodyBytes) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_RazorPay } diff --git a/pkg/detectors/reachmail/reachmail.go b/pkg/detectors/reachmail/reachmail.go index f4797ea95433..7468327ff89a 100644 --- a/pkg/detectors/reachmail/reachmail.go +++ b/pkg/detectors/reachmail/reachmail.go @@ -3,10 +3,11 @@ package reachmail import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/readme/readme.go b/pkg/detectors/readme/readme.go index 910b4ddeb369..7d50e2a28f7e 100644 --- a/pkg/detectors/readme/readme.go +++ b/pkg/detectors/readme/readme.go @@ -2,10 +2,11 @@ package readme import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -50,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("accept", "application/json") res, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems.go b/pkg/detectors/reallysimplesystems/reallysimplesystems.go index ea7506c5e332..4cfd6cad65ce 100644 --- a/pkg/detectors/reallysimplesystems/reallysimplesystems.go +++ b/pkg/detectors/reallysimplesystems/reallysimplesystems.go @@ -49,26 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv4.reallysimplesystems.com/accounts", nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if json.Valid(bodyBytes) { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -77,6 +60,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://apiv4.reallysimplesystems.com/accounts", nil) + if err != nil { + return false, err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + if res.StatusCode >= 200 && res.StatusCode < 300 { + if json.Valid(bodyBytes) { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_ReallySimpleSystems } diff --git a/pkg/detectors/rebrandly/rebrandly.go b/pkg/detectors/rebrandly/rebrandly.go index 7ce567e42198..d830943bca26 100644 --- a/pkg/detectors/rebrandly/rebrandly.go +++ b/pkg/detectors/rebrandly/rebrandly.go @@ -2,10 +2,11 @@ package rebrandly import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("apikey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rechargepayments/rechargepayments.go b/pkg/detectors/rechargepayments/rechargepayments.go index 6cf5282ae185..f6e2c2225d06 100644 --- a/pkg/detectors/rechargepayments/rechargepayments.go +++ b/pkg/detectors/rechargepayments/rechargepayments.go @@ -2,9 +2,10 @@ package rechargepayments import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Recharge-Access-Token", token) res, err := client.Do(req) if err == nil { - res.Body.Close() // The request body is unused. + _ = res.Body.Close() // The request body is unused. if res.StatusCode == http.StatusOK { result.Verified = true diff --git a/pkg/detectors/redhatpyxis/redhatpyxis.go b/pkg/detectors/redhatpyxis/redhatpyxis.go new file mode 100644 index 000000000000..46e8e869fec3 --- /dev/null +++ b/pkg/detectors/redhatpyxis/redhatpyxis.go @@ -0,0 +1,132 @@ +package redhatpyxis + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient()) + + pyxisAPIKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"redhat", "pyxis"}) + `\b([a-z0-9]{32})\b`) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{ + "redhat", + "pyxis", + } +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for Red Hat Pyxis API keys and optionally verifies them. +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range pyxisAPIKeyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_RedHatPyxis, + Raw: []byte(token), + SecretParts: map[string]string{ + "key": token, + }, + } + + if verify { + verified, verificationErr := verifyPyxisAPIKey( + ctx, + s.getClient(), + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + + return +} + +func verifyPyxisAPIKey( + ctx context.Context, + client *http.Client, + token string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://catalog.redhat.com/api/containers/v1/projects/certification/requests/images?page_size=1&page=0", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("X-API-KEY", token) + + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_RedHatPyxis, token, req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + + case http.StatusUnauthorized: + // Invalid API key + return false, nil + + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_RedHatPyxis +} + +func (s Scanner) Description() string { + return "Red Hat Pyxis is a container certification and metadata management platform. Red Hat Pyxis API keys can be used to authenticate against the Red Hat Ecosystem Catalog APIs and access container certification resources and metadata." +} diff --git a/pkg/detectors/redhatpyxis/redhatpyxis_integration_test.go b/pkg/detectors/redhatpyxis/redhatpyxis_integration_test.go new file mode 100644 index 000000000000..395355eb8269 --- /dev/null +++ b/pkg/detectors/redhatpyxis/redhatpyxis_integration_test.go @@ -0,0 +1,211 @@ +//go:build detectors +// +build detectors + +package redhatpyxis + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestRedHatPyxis_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeToken := testSecrets.MustGetField("REDHAT_PYXIS_API_KEY") + + // Random inactive token matching the expected format + inactiveToken := "o9ynnj1wfw33a50g9009ti0ne1kqe8ac" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "redhat api key %s", + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_RedHatPyxis, + Verified: true, + Raw: []byte(activeToken), + }, + }, + }, + { + name: "found, real token, verification error due to timeout", + s: Scanner{ + client: common.SaneHttpClientTimeOut(1 * time.Microsecond), + }, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "redhat api key %s", + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_RedHatPyxis, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, verification error due to unexpected api surface", + s: Scanner{ + client: common.ConstantResponseHttpClient(500, "{}"), + }, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "redhat api key %s", + activeToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_RedHatPyxis, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf( + []byte{}, + "redhat api key %s", + inactiveToken, + ), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_RedHatPyxis, + Verified: false, + Raw: []byte(inactiveToken), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData( + tt.args.ctx, + tt.args.verify, + tt.args.data, + ) + + if (err != nil) != tt.wantErr { + t.Fatalf( + "RedHatPyxis.FromData() error = %v, wantErr %v", + err, + tt.wantErr, + ) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf( + "RedHatPyxis.FromData() %s diff: (-got +want)\n%s", + tt.name, + diff, + ) + } + }) + } +} + +func BenchmarkRedHatPyxis_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/redhatpyxis/redhatpyxis_test.go b/pkg/detectors/redhatpyxis/redhatpyxis_test.go new file mode 100644 index 000000000000..3bd14cc9dc22 --- /dev/null +++ b/pkg/detectors/redhatpyxis/redhatpyxis_test.go @@ -0,0 +1,129 @@ +package redhatpyxis + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestRedHatPyxis_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - basic", + input: ` + [INFO] redhat integration initialized + [DEBUG] redhat_api_key=o9ynnj1wfw33a50g9009ti0ne1kqe8ac + `, + want: []string{ + "o9ynnj1wfw33a50g9009ti0ne1kqe8ac", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + redhat nvropikx1e0vzxpad8upkf8696icmidx + redhatpyxis u0oxtxaurp1lbsagehya1rk6r9cxcc7f + `, + want: []string{ + "nvropikx1e0vzxpad8upkf8696icmidx", + "u0oxtxaurp1lbsagehya1rk6r9cxcc7f", + }, + }, + { + name: "invalid pattern - uppercase characters", + input: ` + redhat O9YNNJ1WFW33A50G9009TI0NE1KQE8AC + `, + want: nil, + }, + { + name: "invalid pattern - too short", + input: ` + redhat abc123 + `, + want: nil, + }, + { + name: "invalid pattern - too long", + input: ` + redhat o9ynnj1wfw33a50g9009ti0ne1kqe8acabc + `, + want: nil, + }, + { + name: "invalid pattern - no keyword nearby", + input: ` + o9ynnj1wfw33a50g9009ti0ne1kqe8ac + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + redhat api key configured + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + + if len(test.want) > 0 && len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData( + context.Background(), + false, + []byte(test.input), + ) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.go index 4c4c39f73747..2c6fb70ee36b 100644 --- a/pkg/detectors/redis/redis.go +++ b/pkg/detectors/redis/redis.go @@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } - redact := strings.TrimSpace(strings.Replace(urlMatch, password, "*******", -1)) + redact := strings.TrimSpace(strings.ReplaceAll(urlMatch, password, "*******")) s := detectors.Result{ DetectorType: detector_typepb.DetectorType_Redis, @@ -65,7 +65,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "host": parsedURL.Host, "password": password, }, - Redacted: redact, + Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -99,7 +100,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } - redact := strings.TrimSpace(strings.Replace(urlMatch, password, "*******", -1)) + redact := strings.TrimSpace(strings.ReplaceAll(urlMatch, password, "*******")) s := detectors.Result{ DetectorType: detector_typepb.DetectorType_Redis, @@ -108,7 +109,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result "host": parsedURL.Host, "password": password, }, - Redacted: redact, + Redacted: redact, + ExtraData: extraDataFromURL(parsedURL), } if verify { @@ -128,6 +130,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func extraDataFromURL(u *url.URL) map[string]string { + extraData := make(map[string]string) + if u.Host != "" { + extraData["host"] = u.Host + } + if u.User != nil && u.User.Username() != "" { + extraData["username"] = u.User.Username() + } + return extraData +} + func verifyRedis(ctx context.Context, u *url.URL) bool { opt, err := redis.ParseURL(u.String()) if err != nil { diff --git a/pkg/detectors/redis/redis_integration_test.go b/pkg/detectors/redis/redis_integration_test.go index 44380084e700..8f21d3c720f5 100644 --- a/pkg/detectors/redis/redis_integration_test.go +++ b/pkg/detectors/redis/redis_integration_test.go @@ -147,6 +147,8 @@ func TestRedisIntegration_FromChunk(t *testing.T) { } for i := range got { got[i].Raw = nil + got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) @@ -206,6 +208,8 @@ func TestURI_FromChunk(t *testing.T) { } for i := range got { got[i].Raw = nil + got[i].SecretParts = nil + got[i].ExtraData = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go index fccd171ab2c2..c9050a9211ae 100644 --- a/pkg/detectors/redis/redis_test.go +++ b/pkg/detectors/redis/redis_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" @@ -22,6 +24,46 @@ var ( keyword = "redis" ) +func TestRedis_ExtraData(t *testing.T) { + tests := []struct { + name string + data string + wantHost string + wantUsername string + }{ + { + name: "standard redis URI", + data: `redis://myuser:mysecretpass@redis.example.com:6379/0`, + wantHost: "redis.example.com:6379", + wantUsername: "myuser", + }, + { + name: "redis URI with default username", + data: `redis://default:mysecretpass@redis.example.com:6379`, + wantHost: "redis.example.com:6379", + wantUsername: "default", + }, + { + name: "azure redis pattern without username", + data: `mycache.redis.cache.windows.net:6380,password=Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS,ssl=True,abortConnect=False`, + wantHost: "mycache.redis.cache.windows.net:6380", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + results, err := s.FromData(context.Background(), false, []byte(tt.data)) + require.NoError(t, err) + require.NotEmpty(t, results, "expected at least one result") + + r := results[0] + assert.Equal(t, tt.wantHost, r.ExtraData["host"]) + assert.Equal(t, tt.wantUsername, r.ExtraData["username"]) + }) + } +} + func TestRedisIntegration_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) diff --git a/pkg/detectors/refiner/refiner.go b/pkg/detectors/refiner/refiner.go index a226cbdea4cc..c7f689bb4c2b 100644 --- a/pkg/detectors/refiner/refiner.go +++ b/pkg/detectors/refiner/refiner.go @@ -3,10 +3,11 @@ package refiner import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rentman/rentman.go b/pkg/detectors/rentman/rentman.go index d242949239fa..fc0464233060 100644 --- a/pkg/detectors/rentman/rentman.go +++ b/pkg/detectors/rentman/rentman.go @@ -3,10 +3,11 @@ package rentman import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/repairshopr/repairshopr.go b/pkg/detectors/repairshopr/repairshopr.go index 203096388e3c..5bdf9a3e9407 100644 --- a/pkg/detectors/repairshopr/repairshopr.go +++ b/pkg/detectors/repairshopr/repairshopr.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/replicate/replicate.go b/pkg/detectors/replicate/replicate.go index 04ba7298edb4..68923d5d7d31 100644 --- a/pkg/detectors/replicate/replicate.go +++ b/pkg/detectors/replicate/replicate.go @@ -3,10 +3,11 @@ package replicate import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/replyio/replyio.go b/pkg/detectors/replyio/replyio.go index b0740ae00769..8fff1e3ffbd5 100644 --- a/pkg/detectors/replyio/replyio.go +++ b/pkg/detectors/replyio/replyio.go @@ -3,10 +3,11 @@ package replyio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/requestfinance/requestfinance.go b/pkg/detectors/requestfinance/requestfinance.go index 4f10dbd241a9..4216cd8f296e 100644 --- a/pkg/detectors/requestfinance/requestfinance.go +++ b/pkg/detectors/requestfinance/requestfinance.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/resend/resend.go b/pkg/detectors/resend/resend.go new file mode 100644 index 000000000000..ef34931a6195 --- /dev/null +++ b/pkg/detectors/resend/resend.go @@ -0,0 +1,149 @@ +package resend + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +// GET /api-keys verifies both Resend permission tiers with a single +// read-only request: +// - full_access -> 200 +// - sending_access -> 401 with body {"name":"restricted_api_key"} +// +// See https://resend.com/docs/api-reference/api-keys/list-api-keys and +// https://resend.com/docs/api-reference/errors. +const verifyURL = "https://api.resend.com/api-keys" + +var ( + defaultClient = common.SaneHttpClient() + + // Resend API keys: prefix `re_`, 8 chars of base58, + // underscore, then 24 chars from the same alphabet. + keyPat = regexp.MustCompile(`\b(re_[1-9A-HJ-NP-Za-km-z]{8}_[1-9A-HJ-NP-Za-km-z]{24})\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"re_"} +} + +// FromData will find and optionally verify Resend secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueMatches := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueMatches[match[1]] = struct{}{} + } + + for token := range uniqueMatches { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_Resend, + Raw: []byte(token), + SecretParts: map[string]string{"key": token}, + } + + if verify { + client := s.getClient() + isVerified, extraData, verificationErr := verifyMatch(ctx, client, token) + s1.Verified = isVerified + s1.ExtraData = extraData + s1.SetVerificationError(verificationErr, token) + } + + results = append(results, s1) + } + + return results, nil +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +type resendError struct { + Name string `json:"name"` + Message string `json:"message"` +} + +func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, verifyURL, nil) + if err != nil { + return false, nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + res, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + extraData := map[string]string{ + "rotation_guide": "https://resend.com/docs/dashboard/api-keys/introduction", + } + + switch res.StatusCode { + case http.StatusOK: + // Active key with `full_access` permission. + extraData["permission"] = "full_access" + return true, extraData, nil + + case http.StatusUnauthorized: + // Resend returns 401 for two distinct cases. We must look at the + // error name to tell them apart: + // - restricted_api_key: active `sending_access` key trying to + // reach a non-send route. The key is real and can send mail. + // - missing_api_key: no Authorization header (should not + // happen here since we set one above). + var body resendError + if err := json.NewDecoder(res.Body).Decode(&body); err != nil { + return false, nil, fmt.Errorf("failed to decode 401 response body: %w", err) + } + if body.Name == "restricted_api_key" { + extraData["permission"] = "sending_access" + return true, extraData, nil + } + return false, nil, nil + + case http.StatusBadRequest, http.StatusForbidden: + // 400 validation_error: unknown, malformed, or soft-deleted key. + // 403 suspended_api_key / restricted_api_key ("not active"): + // the key exists but is already disabled; no rotation needed. + return false, nil, nil + + default: + return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_Resend +} + +func (s Scanner) Description() string { + return "Resend is an email API for transactional and marketing email. Resend API keys can be used to send email, manage domains and audiences, and access other resources on the account." +} diff --git a/pkg/detectors/resend/resend_integration_test.go b/pkg/detectors/resend/resend_integration_test.go new file mode 100644 index 000000000000..5c58a21c4499 --- /dev/null +++ b/pkg/detectors/resend/resend_integration_test.go @@ -0,0 +1,188 @@ +//go:build detectors +// +build detectors + +package resend + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestResend_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("RESEND_API_KEY") + sendingAccessSecret := testSecrets.MustGetField("RESEND_API_KEY_SENDING_ACCESS") + inactiveSecret := testSecrets.MustGetField("RESEND_API_KEY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified (full_access)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a resend secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Resend, + Verified: true, + ExtraData: map[string]string{ + "permission": "full_access", + "rotation_guide": "https://resend.com/docs/dashboard/api-keys/introduction", + }, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, verified (sending_access)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a resend secret %s within", sendingAccessSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Resend, + Verified: true, + ExtraData: map[string]string{ + "permission": "sending_access", + "rotation_guide": "https://resend.com/docs/dashboard/api-keys/introduction", + }, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a resend secret %s within but not valid", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Resend, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a resend secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Resend, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a resend secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Resend, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Resend.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Errorf("Resend.FromData() error = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "SecretParts") + ignoreUnexported := cmpopts.IgnoreUnexported(detectors.Result{}) + if diff := cmp.Diff(got, tt.want, ignoreOpts, ignoreUnexported); diff != "" { + t.Errorf("Resend.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/resend/resend_test.go b/pkg/detectors/resend/resend_test.go new file mode 100644 index 000000000000..dff632aea2f6 --- /dev/null +++ b/pkg/detectors/resend/resend_test.go @@ -0,0 +1,91 @@ +package resend + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "re_1234abcd_aBcDeFgHiJkLmNoPqRsTuVwX" + invalidPattern = "re_1234abcd_aBcDeFgHiJk0OIlmNoPqRsTu" + keyword = "resend" +) + +func TestResend_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - with keyword resend", + input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern - ignore duplicate", + input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), + want: []string{validPattern}, + }, + { + name: "invalid pattern - contains disallowed base58 chars (0/O/I/l)", + input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), + want: []string{}, + }, + { + name: "invalid pattern - wrong length", + input: fmt.Sprintf("%s = 're_1234abcd_short'", keyword), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi.go index 8f9ec60f61f0..f73545086279 100644 --- a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi.go +++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Access-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi.go index c4e4652de586..4f0bfcbd1e27 100644 --- a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi.go +++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Access-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/result_test.go b/pkg/detectors/result_test.go new file mode 100644 index 000000000000..95baebef983d --- /dev/null +++ b/pkg/detectors/result_test.go @@ -0,0 +1,31 @@ +package detectors + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestClearSecrets(t *testing.T) { + result := Result{ + Raw: []byte("raw-secret"), + RawV2: []byte("raw-secret-v2"), + Redacted: "redacted", + SecretParts: map[string]string{"key": "raw-secret"}, + ExtraData: map[string]string{"account": "123"}, + } + result.SetPrimarySecretValue("secret-value") + result.SetPrimarySecretLine(12) + result.SetVerificationError(assert.AnError) + + result.ClearSecrets() + + assert.Nil(t, result.Raw) + assert.Nil(t, result.RawV2) + assert.Nil(t, result.SecretParts) + assert.Empty(t, result.GetPrimarySecretValue()) + assert.Zero(t, result.primarySecret.Line) + assert.Equal(t, "redacted", result.Redacted) + assert.Equal(t, map[string]string{"account": "123"}, result.ExtraData) + assert.EqualError(t, result.VerificationError(), assert.AnError.Error()) +} diff --git a/pkg/detectors/rev/rev.go b/pkg/detectors/rev/rev.go index 27e7e93c66af..d90295551db4 100644 --- a/pkg/detectors/rev/rev.go +++ b/pkg/detectors/rev/rev.go @@ -3,10 +3,11 @@ package rev import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -49,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Rev, Raw: []byte(resUserMatch), - SecretParts: map[string]string{"key": resUserMatch}, + SecretParts: map[string]string{"key": resUserMatch, "client": resClientMatch}, } if verify { @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Rev %s:%s", resClientMatch, resUserMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rev/rev_integration_test.go b/pkg/detectors/rev/rev_integration_test.go index 3628811c49f8..63b059de3f1d 100644 --- a/pkg/detectors/rev/rev_integration_test.go +++ b/pkg/detectors/rev/rev_integration_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -98,7 +98,17 @@ func TestRev_FromChunk(t *testing.T) { } got[i].Raw = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Rev.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/revampcrm/revampcrm.go b/pkg/detectors/revampcrm/revampcrm.go index 2e4088344a9f..844b17158c44 100644 --- a/pkg/detectors/revampcrm/revampcrm.go +++ b/pkg/detectors/revampcrm/revampcrm.go @@ -2,10 +2,11 @@ package revampcrm import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ringcentral/ringcentral.go b/pkg/detectors/ringcentral/ringcentral.go index 75d7026a91d7..43a2adf31c49 100644 --- a/pkg/detectors/ringcentral/ringcentral.go +++ b/pkg/detectors/ringcentral/ringcentral.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ritekit/ritekit.go b/pkg/detectors/ritekit/ritekit.go index 692a473888bb..18b0288d4001 100644 --- a/pkg/detectors/ritekit/ritekit.go +++ b/pkg/detectors/ritekit/ritekit.go @@ -3,10 +3,11 @@ package ritekit import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/roaring/roaring.go b/pkg/detectors/roaring/roaring.go index e6b3b2b7cce5..6ac68f33bf86 100644 --- a/pkg/detectors/roaring/roaring.go +++ b/pkg/detectors/roaring/roaring.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -68,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rocketreach/rocketreach.go b/pkg/detectors/rocketreach/rocketreach.go index 4db1d9250225..b57e5c991e19 100644 --- a/pkg/detectors/rocketreach/rocketreach.go +++ b/pkg/detectors/rocketreach/rocketreach.go @@ -2,10 +2,11 @@ package rocketreach import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rootly/rootly.go b/pkg/detectors/rootly/rootly.go index 524305a14163..aa3283250e04 100644 --- a/pkg/detectors/rootly/rootly.go +++ b/pkg/detectors/rootly/rootly.go @@ -72,7 +72,7 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK, http.StatusNotFound: diff --git a/pkg/detectors/route4me/route4me.go b/pkg/detectors/route4me/route4me.go index 696576561ef4..b6de75caab78 100644 --- a/pkg/detectors/route4me/route4me.go +++ b/pkg/detectors/route4me/route4me.go @@ -2,10 +2,11 @@ package route4me import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rownd/rownd.go b/pkg/detectors/rownd/rownd.go index 98999f9b3cb7..a65c0622fdeb 100644 --- a/pkg/detectors/rownd/rownd.go +++ b/pkg/detectors/rownd/rownd.go @@ -2,10 +2,11 @@ package rownd import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -70,7 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-rownd-app-secret", secretMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/rubygems/rubygems.go b/pkg/detectors/rubygems/rubygems.go index d7dacbce975c..f8ab042106aa 100644 --- a/pkg/detectors/rubygems/rubygems.go +++ b/pkg/detectors/rubygems/rubygems.go @@ -2,10 +2,11 @@ package rubygems import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == http.StatusForbidden { s1.Verified = true } diff --git a/pkg/detectors/runrunit/runrunit.go b/pkg/detectors/runrunit/runrunit.go index 365e3f8bd8b5..6fce3c8d3ee3 100644 --- a/pkg/detectors/runrunit/runrunit.go +++ b/pkg/detectors/runrunit/runrunit.go @@ -2,10 +2,11 @@ package runrunit import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("User-Token", resUserTokenMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/salesblink/salesblink.go b/pkg/detectors/salesblink/salesblink.go index 5dfac3d26842..771af9669232 100644 --- a/pkg/detectors/salesblink/salesblink.go +++ b/pkg/detectors/salesblink/salesblink.go @@ -2,10 +2,11 @@ package salesblink import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/salescookie/salescookie.go b/pkg/detectors/salescookie/salescookie.go index b30af186c483..eb68e8decefd 100644 --- a/pkg/detectors/salescookie/salescookie.go +++ b/pkg/detectors/salescookie/salescookie.go @@ -2,10 +2,11 @@ package salescookie import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/salesflare/salesflare.go b/pkg/detectors/salesflare/salesflare.go index 41b6ebdd3101..af0fbf2be082 100644 --- a/pkg/detectors/salesflare/salesflare.go +++ b/pkg/detectors/salesflare/salesflare.go @@ -3,10 +3,11 @@ package salesflare import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.go index d9d88dd25ed3..89ca428613b7 100644 --- a/pkg/detectors/salesforce/salesforce.go +++ b/pkg/detectors/salesforce/salesforce.go @@ -82,7 +82,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "records") - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse { s1.Verified = true } else if res.StatusCode >= 200 && res.StatusCode < 300 && !verifiedBodyResponse { diff --git a/pkg/detectors/salesforceoauth2/salesforceoauth2.go b/pkg/detectors/salesforceoauth2/salesforceoauth2.go index dfd995188a66..0880698fd67a 100644 --- a/pkg/detectors/salesforceoauth2/salesforceoauth2.go +++ b/pkg/detectors/salesforceoauth2/salesforceoauth2.go @@ -92,6 +92,11 @@ domainLoop: DetectorType: detector_typepb.DetectorType_SalesforceOauth2, Raw: []byte(secret), RawV2: fmt.Appendf([]byte{}, "%s:%s:%s", domain, key, secret), + SecretParts: map[string]string{ + "domain": domain, + "client_id": key, + "client_secret": secret, + }, } if verify { @@ -105,15 +110,6 @@ domainLoop: s1.SetVerificationError(verificationErr, secret) } - - if isVerified { - s1.SecretParts = map[string]string{ - "domain": domain, - "client_id": key, - "client_secret": secret, - } - - } } results = append(results, s1) diff --git a/pkg/detectors/salesmate/salesmate.go b/pkg/detectors/salesmate/salesmate.go index 1bdadef1daf4..582e334927ae 100644 --- a/pkg/detectors/salesmate/salesmate.go +++ b/pkg/detectors/salesmate/salesmate.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("sessionToken", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go b/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go index 6ab32f061cb2..8b2f1a915aab 100644 --- a/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go +++ b/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go @@ -33,7 +33,7 @@ var ( // Perform the request client := &http.Client{} resp, _ := client.Do(req) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Check response status if resp.StatusCode == http.StatusNoContent { diff --git a/pkg/detectors/scalewaykey/scalewaykey.go b/pkg/detectors/scalewaykey/scalewaykey.go index e6fd1189d7a9..b9eb69217e08 100644 --- a/pkg/detectors/scalewaykey/scalewaykey.go +++ b/pkg/detectors/scalewaykey/scalewaykey.go @@ -2,10 +2,11 @@ package scalewaykey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Auth-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/scalr/scalr.go b/pkg/detectors/scalr/scalr.go index e4ef42b8f333..283d72671861 100644 --- a/pkg/detectors/scalr/scalr.go +++ b/pkg/detectors/scalr/scalr.go @@ -3,10 +3,11 @@ package scalr import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -61,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Prefer", "profile=preview") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/scrapeowl/scrapeowl.go b/pkg/detectors/scrapeowl/scrapeowl.go index a18329bcdd0b..c515d22641c9 100644 --- a/pkg/detectors/scrapeowl/scrapeowl.go +++ b/pkg/detectors/scrapeowl/scrapeowl.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/scraperapi/scraperapi.go b/pkg/detectors/scraperapi/scraperapi.go index 96f93691fb1e..e291a3147a57 100644 --- a/pkg/detectors/scraperapi/scraperapi.go +++ b/pkg/detectors/scraperapi/scraperapi.go @@ -3,11 +3,12 @@ package scraperapi import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/scraperbox/scraperbox.go b/pkg/detectors/scraperbox/scraperbox.go index 4a12dbad61df..65f9e8220c7f 100644 --- a/pkg/detectors/scraperbox/scraperbox.go +++ b/pkg/detectors/scraperbox/scraperbox.go @@ -3,11 +3,12 @@ package scraperbox import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/scrapestack/scrapestack.go b/pkg/detectors/scrapestack/scrapestack.go index 50c03e419b17..adf909b4db17 100644 --- a/pkg/detectors/scrapestack/scrapestack.go +++ b/pkg/detectors/scrapestack/scrapestack.go @@ -3,11 +3,12 @@ package scrapestack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `html`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/scrapfly/scrapfly.go b/pkg/detectors/scrapfly/scrapfly.go index 780f71963d94..92dc9f9f8046 100644 --- a/pkg/detectors/scrapfly/scrapfly.go +++ b/pkg/detectors/scrapfly/scrapfly.go @@ -3,11 +3,12 @@ package scrapfly import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/scrapingant/scrapingant.go b/pkg/detectors/scrapingant/scrapingant.go index 75fade77e78e..312aec86b573 100644 --- a/pkg/detectors/scrapingant/scrapingant.go +++ b/pkg/detectors/scrapingant/scrapingant.go @@ -89,11 +89,12 @@ func verifyScrapingAnt(ctx context.Context, client *http.Client, apiKey string) _ = resp.Body.Close() }() - if resp.StatusCode == http.StatusOK { + switch resp.StatusCode { + case http.StatusOK: return true, nil - } else if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + case http.StatusUnauthorized, http.StatusForbidden: return false, nil - } else { + default: return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } } diff --git a/pkg/detectors/screenshotapi/screenshotapi.go b/pkg/detectors/screenshotapi/screenshotapi.go index d4df2c733495..8d10a2b0ce94 100644 --- a/pkg/detectors/screenshotapi/screenshotapi.go +++ b/pkg/detectors/screenshotapi/screenshotapi.go @@ -3,11 +3,12 @@ package screenshotapi import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() var r response if err := json.NewDecoder(res.Body).Decode(&r); err != nil { s1.SetVerificationError(err, resMatch) diff --git a/pkg/detectors/screenshotlayer/screenshotlayer.go b/pkg/detectors/screenshotlayer/screenshotlayer.go index 346124a69fa8..6c98c8b5b810 100644 --- a/pkg/detectors/screenshotlayer/screenshotlayer.go +++ b/pkg/detectors/screenshotlayer/screenshotlayer.go @@ -3,12 +3,13 @@ package screenshotlayer import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `PNG`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/scrutinizerci/scrutinizerci.go b/pkg/detectors/scrutinizerci/scrutinizerci.go index 61ab7ff469b9..4c32c7c1b3cb 100644 --- a/pkg/detectors/scrutinizerci/scrutinizerci.go +++ b/pkg/detectors/scrutinizerci/scrutinizerci.go @@ -2,10 +2,11 @@ package scrutinizerci import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/securitytrails/securitytrails.go b/pkg/detectors/securitytrails/securitytrails.go index d3faffc7a5e8..a8bf7f8be81b 100644 --- a/pkg/detectors/securitytrails/securitytrails.go +++ b/pkg/detectors/securitytrails/securitytrails.go @@ -2,10 +2,11 @@ package securitytrails import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("APIKEY", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/segmentapikey/segmentapikey.go b/pkg/detectors/segmentapikey/segmentapikey.go index e7c3c92fbc27..935b43adaf05 100644 --- a/pkg/detectors/segmentapikey/segmentapikey.go +++ b/pkg/detectors/segmentapikey/segmentapikey.go @@ -3,10 +3,11 @@ package segmentapikey import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/selectpdf/selectpdf.go b/pkg/detectors/selectpdf/selectpdf.go index 400b7f6b5928..e456b9ef716c 100644 --- a/pkg/detectors/selectpdf/selectpdf.go +++ b/pkg/detectors/selectpdf/selectpdf.go @@ -3,11 +3,12 @@ package selectpdf import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/semaphore/semaphore.go b/pkg/detectors/semaphore/semaphore.go index 874c36b24ffb..8d50be925c5d 100644 --- a/pkg/detectors/semaphore/semaphore.go +++ b/pkg/detectors/semaphore/semaphore.go @@ -3,11 +3,12 @@ package semaphore import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,23 +52,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.semaphore.co/api/v4/account?apikey=%s", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/vnd.semaphore+json; version=3") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "account_id") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -76,6 +63,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.semaphore.co/api/v4/account?apikey=%s", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Accept", "application/vnd.semaphore+json; version=3") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "account_id") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Semaphore } diff --git a/pkg/detectors/sendbird/sendbird.go b/pkg/detectors/sendbird/sendbird.go index 3785de53e5e7..026cff7a78ac 100644 --- a/pkg/detectors/sendbird/sendbird.go +++ b/pkg/detectors/sendbird/sendbird.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -75,7 +76,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 400 { // Sendbird returns 400 for all errors diff --git a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi.go b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi.go index 10b265e989b8..f0e3e320b11a 100644 --- a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi.go +++ b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi.go @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { s1.SetVerificationError(err, resMatch) } else { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode != http.StatusForbidden { diff --git a/pkg/detectors/sendgrid/sendgrid.go b/pkg/detectors/sendgrid/sendgrid.go index acb843104c17..852935bbc993 100644 --- a/pkg/detectors/sendgrid/sendgrid.go +++ b/pkg/detectors/sendgrid/sendgrid.go @@ -48,6 +48,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_SendGrid, Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } if verify { @@ -60,7 +61,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = verified s1.ExtraData = extraData s1.SetVerificationError(verificationErr) - s1.SecretParts = map[string]string{"key": token} } results = append(results, s1) diff --git a/pkg/detectors/sendinbluev2/sendinbluev2.go b/pkg/detectors/sendinbluev2/sendinbluev2.go index 63508b2ea849..fb16ed94d26e 100644 --- a/pkg/detectors/sendinbluev2/sendinbluev2.go +++ b/pkg/detectors/sendinbluev2/sendinbluev2.go @@ -2,10 +2,11 @@ package sendinbluev2 import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/serphouse/serphouse.go b/pkg/detectors/serphouse/serphouse.go index 772e0470fe4d..1832f4e39e84 100644 --- a/pkg/detectors/serphouse/serphouse.go +++ b/pkg/detectors/serphouse/serphouse.go @@ -3,10 +3,11 @@ package serphouse import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/serpstack/serpstack.go b/pkg/detectors/serpstack/serpstack.go index 79c905ae06b7..65ea2cbf03f2 100644 --- a/pkg/detectors/serpstack/serpstack.go +++ b/pkg/detectors/serpstack/serpstack.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `search_url`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/sheety/sheety.go b/pkg/detectors/sheety/sheety.go index 46f409a52b3b..13bb3f20abc6 100644 --- a/pkg/detectors/sheety/sheety.go +++ b/pkg/detectors/sheety/sheety.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sherpadesk/sherpadesk.go b/pkg/detectors/sherpadesk/sherpadesk.go index 0ec99b5a3e44..ff9e1f91fa1d 100644 --- a/pkg/detectors/sherpadesk/sherpadesk.go +++ b/pkg/detectors/sherpadesk/sherpadesk.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/shipday/shipday.go b/pkg/detectors/shipday/shipday.go index 82434b87ae15..e1e0796edbb5 100644 --- a/pkg/detectors/shipday/shipday.go +++ b/pkg/detectors/shipday/shipday.go @@ -3,10 +3,11 @@ package shipday import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/shippo/shippo.go b/pkg/detectors/shippo/shippo.go new file mode 100644 index 000000000000..b0d7ff312c7a --- /dev/null +++ b/pkg/detectors/shippo/shippo.go @@ -0,0 +1,132 @@ +package shippo + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // Shippo live tokens: + // Format: shippo_live_ or shippo_test_ + 40 hex characters + shippoLiveTokenPat = regexp.MustCompile( + `\b(shippo_(live|test)_[a-f0-9]{40})\b`, + ) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{"shippo_live_", "shippo_test_"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for Shippo live API tokens and optionally verifies them +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range shippoLiveTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_Shippo, + Raw: []byte(token), + Redacted: token[:12] + "...", + SecretParts: map[string]string{ + "key": token, + }, + } + + if verify { + verified, verificationErr := verifyShippoToken( + ctx, + s.getClient(), + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + + return +} + +func verifyShippoToken( + ctx context.Context, + client *http.Client, + token string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://api.goshippo.com/shippo-accounts?page=1&results=1", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "ShippoToken "+token) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + // Token invalid or revoked + return false, nil + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_Shippo +} + +func (s Scanner) Description() string { + return "Shippo is a shipping API platform. Shippo live API tokens can be used to authenticate API requests and manage shipping operations." +} diff --git a/pkg/detectors/shippo/shippo_integration_test.go b/pkg/detectors/shippo/shippo_integration_test.go new file mode 100644 index 000000000000..3d100e398509 --- /dev/null +++ b/pkg/detectors/shippo/shippo_integration_test.go @@ -0,0 +1,182 @@ +//go:build detectors +// +build detectors + +package shippo + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestShippo_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // Secrets stored in GCP test project + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeToken := testSecrets.MustGetField("SHIPPO_TEST_TOKEN") + inactiveToken := "shippo_live_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Shippo API token %s for shipment", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Shippo, + Verified: true, + Raw: []byte(activeToken), + Redacted: activeToken[:12] + "...", + }, + }, + }, + { + name: "found, real token, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Shippo API token %s for shipment", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Shippo, + Verified: false, + Raw: []byte(activeToken), + Redacted: activeToken[:12] + "...", + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Shippo API token %s for shipment", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Shippo, + Verified: false, + Raw: []byte(activeToken), + Redacted: activeToken[:12] + "...", + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Shippo API token %s for shipment", inactiveToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_Shippo, + Verified: false, + Raw: []byte(inactiveToken), + Redacted: inactiveToken[:12] + "...", + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("Shippo.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "Redacted", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Shippo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkShippo_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/shippo/shippo_test.go b/pkg/detectors/shippo/shippo_test.go new file mode 100644 index 000000000000..47e48a859316 --- /dev/null +++ b/pkg/detectors/shippo/shippo_test.go @@ -0,0 +1,240 @@ +package shippo + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestShippo_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - basic", + input: ` + [INFO] Starting shipment service + [DEBUG] token=shippo_live_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + [INFO] Ready + `, + want: []string{ + "shippo_live_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + { + name: "valid pattern - with keyword nearby", + input: ` + [DEBUG] SHIPPO_API_KEY=shippo_live_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + `, + want: []string{ + "shippo_live_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + shippo_live_1111111111111111111111111111111111111111 + shippo_live_2222222222222222222222222222222222222222 + `, + want: []string{ + "shippo_live_1111111111111111111111111111111111111111", + "shippo_live_2222222222222222222222222222222222222222", + }, + }, + { + name: "invalid pattern - uppercase characters", + input: ` + shippo_live_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + `, + want: nil, + }, + { + name: "invalid pattern - too short", + input: ` + shippo_live_1234 + `, + want: nil, + }, + { + name: "invalid pattern - invalid token length", + input: ` + shippo_live_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + [INFO] initializing shippo service shippo_live_ + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + +func TestShippo_TestKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid test key - basic", + input: ` + shippo_test_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + `, + want: []string{ + "shippo_test_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + { + name: "valid test key - with keyword nearby", + input: ` + SHIPPO_API_KEY=shippo_test_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + `, + want: []string{ + "shippo_test_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + }, + { + name: "valid test key - multiple tokens", + input: ` + shippo_test_1111111111111111111111111111111111111111 + shippo_test_2222222222222222222222222222222222222222 + `, + want: []string{ + "shippo_test_1111111111111111111111111111111111111111", + "shippo_test_2222222222222222222222222222222222222222", + }, + }, + { + name: "invalid test key - uppercase characters", + input: ` + shippo_test_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + `, + want: nil, + }, + { + name: "invalid test key - too short", + input: ` + shippo_test_1234 + `, + want: nil, + }, + { + name: "invalid test key - invalid token length", + input: ` + shippo_test_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + `, + want: nil, + }, + { + name: "invalid test key - keyword only", + input: ` + shippo_test_ + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/shodankey/shodankey.go b/pkg/detectors/shodankey/shodankey.go index 1b56b3586d1f..ffd38ee48ce2 100644 --- a/pkg/detectors/shodankey/shodankey.go +++ b/pkg/detectors/shodankey/shodankey.go @@ -3,11 +3,12 @@ package shodankey import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -81,7 +82,7 @@ func verifyToken(ctx context.Context, client *http.Client, token string) bool { if err != nil { return false } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode < 200 || res.StatusCode >= 300 { return false diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index 6bd870926efe..ceee223f140b 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -51,6 +51,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_Shopify, Redacted: domainRes, Raw: []byte(key + domainRes), + SecretParts: map[string]string{ + "key": key, + "store_url": domainRes, + }, } // set key as the primary secret for engine to find the line number @@ -77,12 +81,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData = map[string]string{ "access_scopes": strings.Join(handleArray, ","), } - s1.SecretParts = map[string]string{ - "key": key, - "store_url": domainRes, - } } - res.Body.Close() + _ = res.Body.Close() } } } diff --git a/pkg/detectors/shotstack/shotstack.go b/pkg/detectors/shotstack/shotstack.go index 3735ca167143..d4a8bf51148f 100644 --- a/pkg/detectors/shotstack/shotstack.go +++ b/pkg/detectors/shotstack/shotstack.go @@ -2,10 +2,11 @@ package shotstack import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -84,7 +85,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/shutterstock/shutterstock.go b/pkg/detectors/shutterstock/shutterstock.go index cdd6f116b1be..3e1abccdf48c 100644 --- a/pkg/detectors/shutterstock/shutterstock.go +++ b/pkg/detectors/shutterstock/shutterstock.go @@ -2,10 +2,11 @@ package shutterstock import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(resMatch, resSecretMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth.go b/pkg/detectors/shutterstockoauth/shutterstockoauth.go index 61937ec4562b..4f955088a27c 100644 --- a/pkg/detectors/shutterstockoauth/shutterstockoauth.go +++ b/pkg/detectors/shutterstockoauth/shutterstockoauth.go @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/signable/signable.go b/pkg/detectors/signable/signable.go index cf10728ba1cf..4c6b5da0d29c 100644 --- a/pkg/detectors/signable/signable.go +++ b/pkg/detectors/signable/signable.go @@ -99,7 +99,7 @@ func verifyResult(ctx context.Context, client *http.Client, token string) (bool, return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { return true, nil } diff --git a/pkg/detectors/signalwire/signalwire.go b/pkg/detectors/signalwire/signalwire.go index 87ce6eae21f8..1e86e714d894 100644 --- a/pkg/detectors/signalwire/signalwire.go +++ b/pkg/detectors/signalwire/signalwire.go @@ -69,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/signaturit/signaturit.go b/pkg/detectors/signaturit/signaturit.go index e941bc4a39a9..a51327987fa8 100644 --- a/pkg/detectors/signaturit/signaturit.go +++ b/pkg/detectors/signaturit/signaturit.go @@ -3,10 +3,11 @@ package signaturit import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/signupgenius/signupgenius.go b/pkg/detectors/signupgenius/signupgenius.go index 0c1dc437097f..afe61c142466 100644 --- a/pkg/detectors/signupgenius/signupgenius.go +++ b/pkg/detectors/signupgenius/signupgenius.go @@ -2,10 +2,11 @@ package signupgenius import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sigopt/sigopt.go b/pkg/detectors/sigopt/sigopt.go index 68d7c176d4d2..120385557580 100644 --- a/pkg/detectors/sigopt/sigopt.go +++ b/pkg/detectors/sigopt/sigopt.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/simfin/simfin.go b/pkg/detectors/simfin/simfin.go index 8a079ac2a1d3..3f2558e0c8e1 100644 --- a/pkg/detectors/simfin/simfin.go +++ b/pkg/detectors/simfin/simfin.go @@ -48,24 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://simfin.com/api/v2/companies/list?api-key=%s", resMatch), nil) - if err != nil { - continue - } - - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(bodyBytes) - validResponse := !strings.Contains(bodyString, `"error"`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -74,6 +59,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://simfin.com/api/v2/companies/list?api-key=%s", resMatch), nil) + if err != nil { + return false, err + } + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(bodyBytes) + validResponse := !strings.Contains(bodyString, `"error"`) + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_SimFin } diff --git a/pkg/detectors/simplesat/simplesat.go b/pkg/detectors/simplesat/simplesat.go index d5ea5f21696d..4ba46e532248 100644 --- a/pkg/detectors/simplesat/simplesat.go +++ b/pkg/detectors/simplesat/simplesat.go @@ -2,10 +2,11 @@ package simplesat import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Simplesat-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/simplynoted/simplynoted.go b/pkg/detectors/simplynoted/simplynoted.go index fca259febebd..f7c987f8940c 100644 --- a/pkg/detectors/simplynoted/simplynoted.go +++ b/pkg/detectors/simplynoted/simplynoted.go @@ -3,10 +3,11 @@ package simplynoted import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/simvoly/simvoly.go b/pkg/detectors/simvoly/simvoly.go index b4ba6fcb2279..d8f891a24d1e 100644 --- a/pkg/detectors/simvoly/simvoly.go +++ b/pkg/detectors/simvoly/simvoly.go @@ -3,10 +3,11 @@ package simvoly import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sinchmessage/sinchmessage.go b/pkg/detectors/sinchmessage/sinchmessage.go index b1f84055822f..0d6516f0c5c3 100644 --- a/pkg/detectors/sinchmessage/sinchmessage.go +++ b/pkg/detectors/sinchmessage/sinchmessage.go @@ -3,10 +3,11 @@ package sinchmessage import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -66,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sirv/sirv.go b/pkg/detectors/sirv/sirv.go index 21f135bbbc3d..12cdfa93a827 100644 --- a/pkg/detectors/sirv/sirv.go +++ b/pkg/detectors/sirv/sirv.go @@ -3,11 +3,12 @@ package sirv import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -64,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/siteleaf/siteleaf.go b/pkg/detectors/siteleaf/siteleaf.go index f53d506af7a9..d8a3544ee97b 100644 --- a/pkg/detectors/siteleaf/siteleaf.go +++ b/pkg/detectors/siteleaf/siteleaf.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -65,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/skrappio/skrappio.go b/pkg/detectors/skrappio/skrappio.go index 500c98f0de96..f248d20dfbf3 100644 --- a/pkg/detectors/skrappio/skrappio.go +++ b/pkg/detectors/skrappio/skrappio.go @@ -2,10 +2,11 @@ package skrappio import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Access-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/slack/slack.go b/pkg/detectors/slack/slack.go index 1a5cb9513469..2f85716a7202 100644 --- a/pkg/detectors/slack/slack.go +++ b/pkg/detectors/slack/slack.go @@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Slack, Raw: []byte(token), + SecretParts: map[string]string{"key": token}, } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/", @@ -82,7 +83,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() var authResponse authRes if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil { err = fmt.Errorf("failed to decode auth response: %w", err) @@ -113,9 +114,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } else { s1.SetVerificationError(err, token) } - s1.SecretParts = map[string]string{ - "key": token, - } } results = append(results, s1) diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.go index c81c33fc1607..e4955f212d36 100644 --- a/pkg/detectors/slackwebhook/slackwebhook.go +++ b/pkg/detectors/slackwebhook/slackwebhook.go @@ -63,41 +63,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result client = defaultClient } - // We don't want to actually send anything to webhooks we find. To verify them without spamming them, we - // send an intentionally malformed message and look for a particular expected error message. - payload := strings.NewReader(`intentionally malformed JSON from TruffleHog scan`) - req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - - defer res.Body.Close() - - switch { - case res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusMultipleChoices: - // Hopefully this never happens - it means we actually sent something to a channel somewhere. But - // we at least know the secret is verified. - s1.Verified = true - case res.StatusCode == http.StatusBadRequest && bytes.Equal(bodyBytes, []byte("invalid_payload")): - s1.Verified = true - case res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusForbidden: - // Not a real webhook or the owning app's OAuth token has been revoked or the app has been deleted - // You might want to handle this case or log it. - default: - err = fmt.Errorf("unexpected HTTP response status %d: %s", res.StatusCode, bodyBytes) - s1.SetVerificationError(err, resMatch) - } - } else { - s1.SetVerificationError(err, resMatch) - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -107,6 +75,47 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + // We don't want to actually send anything to webhooks we find. To verify them without spamming them, we + // send an intentionally malformed message and look for a particular expected error message. + payload := strings.NewReader(`intentionally malformed JSON from TruffleHog scan`) + req, err := http.NewRequestWithContext(ctx, "POST", resMatch, payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + switch { + case res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusMultipleChoices: + // Hopefully this never happens - it means we actually sent something to a channel somewhere. But + // we at least know the secret is verified. + return true, nil + case res.StatusCode == http.StatusBadRequest && bytes.Equal(bodyBytes, []byte("invalid_payload")): + return true, nil + case res.StatusCode == http.StatusBadRequest && bytes.Contains(bodyBytes, []byte("invalid_token")): + // Slack may return the bare error code or embed it in a longer body, hence Contains not Equal. + // Revoked webhook / gone workspace or channel is determinate verified=false. + return false, nil + case res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusForbidden: + // Not a real webhook or the owning app's OAuth token has been revoked or the app has been deleted + // You might want to handle this case or log it. + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d: %s", res.StatusCode, bodyBytes) + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_SlackWebhook } diff --git a/pkg/detectors/slackwebhook/slackwebhook_test.go b/pkg/detectors/slackwebhook/slackwebhook_test.go index 1a63f0c8641b..5bbbded1c4ee 100644 --- a/pkg/detectors/slackwebhook/slackwebhook_test.go +++ b/pkg/detectors/slackwebhook/slackwebhook_test.go @@ -2,10 +2,12 @@ package slackwebhook import ( "context" + "net/http" "testing" "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -78,3 +80,61 @@ func TestSlackWebHook_Pattern(t *testing.T) { }) } } + +func TestSlackWebhook_InvalidTokenVerification(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantVerified bool + wantVerificationErr bool + }{ + { + name: "invalid_token is determinate not-live", + statusCode: http.StatusBadRequest, + body: "invalid_token", + wantVerified: false, + wantVerificationErr: false, + }, + { + name: "invalid_token substring is determinate not-live", + statusCode: http.StatusBadRequest, + body: "error: invalid_token for webhook", + wantVerified: false, + wantVerificationErr: false, + }, + { + name: "unexpected 400 remains indeterminate", + statusCode: http.StatusBadRequest, + body: "something_else", + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "invalid_payload is verified", + statusCode: http.StatusBadRequest, + body: "invalid_payload", + wantVerified: true, + wantVerificationErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{client: common.ConstantResponseHttpClient(tt.statusCode, tt.body)} + got, err := s.FromData(context.Background(), true, []byte(validPattern)) + if err != nil { + t.Fatalf("FromData error: %v", err) + } + if len(got) != 1 { + t.Fatalf("expected 1 result, got %d", len(got)) + } + if got[0].Verified != tt.wantVerified { + t.Errorf("Verified = %v, want %v", got[0].Verified, tt.wantVerified) + } + if (got[0].VerificationError() != nil) != tt.wantVerificationErr { + t.Errorf("VerificationError = %v, wantVerificationErr %v", got[0].VerificationError(), tt.wantVerificationErr) + } + }) + } +} diff --git a/pkg/detectors/smartystreets/smartystreets.go b/pkg/detectors/smartystreets/smartystreets.go index 4e7228f72a59..9cea9e76417d 100644 --- a/pkg/detectors/smartystreets/smartystreets.go +++ b/pkg/detectors/smartystreets/smartystreets.go @@ -3,10 +3,11 @@ package smartystreets import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/smooch/smooch.go b/pkg/detectors/smooch/smooch.go index 85209aecc366..5a2dd9db2def 100644 --- a/pkg/detectors/smooch/smooch.go +++ b/pkg/detectors/smooch/smooch.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -68,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/snipcart/snipcart.go b/pkg/detectors/snipcart/snipcart.go index ebcb5733418d..ba9b8432a03d 100644 --- a/pkg/detectors/snipcart/snipcart.go +++ b/pkg/detectors/snipcart/snipcart.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/snowflake/snowflake.go b/pkg/detectors/snowflake/snowflake.go index 7466baf0404a..5e302237638e 100644 --- a/pkg/detectors/snowflake/snowflake.go +++ b/pkg/detectors/snowflake/snowflake.go @@ -29,8 +29,11 @@ var ( // accountIdentifierPat matches Snowflake account identifiers in the format: XXXXXXX-XXXXX // Example: ABC1234-EXAMPLE accountIdentifierPat = regexp.MustCompile(detectors.PrefixRegex([]string{"account"}) + `\b([a-zA-Z]{7}-[0-9a-zA-Z-_]{1,255}(.privatelink)?)\b`) - // usernameExclusionPat defines characters that should not be present in usernames - usernameExclusionPat = `!@#$%^&*{}:<>,.;?()/\+=\s\n` + + // exclude characters that should not be present in usernames + usernameRegexState = common.UsernameRegexCheck(`!@#$%^&*{}:<>,.;?()/\+=\s\n`) + + passwordRegexState = common.PasswordRegexCheck(" \r\n") // Exclude spaces, carriage returns, and line feeds ) const ( @@ -100,13 +103,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return nil, nil } - usernameRegexState := common.UsernameRegexCheck(usernameExclusionPat) usernameMatches := usernameRegexState.Matches(data) if len(usernameMatches) == 0 { return nil, nil } - passwordRegexState := common.PasswordRegexCheck(" \r\n") // Exclude spaces, carriage returns, and line feeds passwordMatches := passwordRegexState.Matches(data) if len(passwordMatches) == 0 { return nil, nil @@ -173,7 +174,7 @@ func verifyMatch(ctx context.Context, account, username, password string) (bool, if err != nil { return false, fmt.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { diff --git a/pkg/detectors/snykkey/snykkey.go b/pkg/detectors/snykkey/snykkey.go index afe4afdd6157..3ae0995cfab9 100644 --- a/pkg/detectors/snykkey/snykkey.go +++ b/pkg/detectors/snykkey/snykkey.go @@ -80,7 +80,8 @@ func (s Scanner) doVerification(ctx context.Context, token string) (bool, map[st _, _ = io.Copy(io.Discard, res.Body) _ = res.Body.Close() }() - if res.StatusCode == http.StatusOK { + switch res.StatusCode { + case http.StatusOK: userDetails := userDetailsResponse{} err := json.NewDecoder(res.Body).Decode(&userDetails) if err != nil { @@ -103,10 +104,10 @@ func (s Scanner) doVerification(ctx context.Context, token string) (bool, map[st extraData["Organizations"] = strings.Join(orgs, ",") } return true, extraData, nil - } else if res.StatusCode == http.StatusUnauthorized { + case http.StatusUnauthorized: // The secret is determinately not verified (nothing to do) return false, nil, nil - } else { + default: err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) return false, nil, err } diff --git a/pkg/detectors/solarwindsobservability/solarwindsobservability.go b/pkg/detectors/solarwindsobservability/solarwindsobservability.go new file mode 100644 index 000000000000..e81c636d72ee --- /dev/null +++ b/pkg/detectors/solarwindsobservability/solarwindsobservability.go @@ -0,0 +1,115 @@ +package solarwindsobservability + +import ( + "context" + "fmt" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +var _ detectors.Detector = (*Scanner)(nil) + +var regions = []string{"na-01", "na-02", "eu-01", "ap-01"} + +var ( + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"solarwinds"}) + `\b([0-9a-zA-Z_-]{71})\b`) +) + +func (s Scanner) Keywords() []string { + return []string{"solarwinds"} +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_SolarWindsObservability, + Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + isVerified, region, verificationErr := verifySolarWindsObservability(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + if region != "" { + s1.SecretParts["region"] = region + s1.ExtraData = map[string]string{"region": region} + } + } + + results = append(results, s1) + } + + return results, nil +} + +// verifySolarWindsObservability tries the token against each regional endpoint and returns the +// region it verified against, if any. +func verifySolarWindsObservability(ctx context.Context, client *http.Client, token string) (bool, string, error) { + var lastErr error + for _, region := range regions { + url := fmt.Sprintf("https://api.%s.cloud.solarwinds.com/v1/metrics", region) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + lastErr = err + continue + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + + res, err := client.Do(req) + if err != nil { + lastErr = err + continue + } + + verified, regionErr := func() (bool, error) { + defer func() { _ = res.Body.Close() }() + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d from region %s", res.StatusCode, region) + } + }() + + if verified { + return true, region, nil + } + if regionErr != nil { + lastErr = regionErr + } + } + + return false, "", lastErr +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_SolarWindsObservability +} + +func (s Scanner) Description() string { + return "SolarWinds Observability is a cloud-based SaaS observability platform (successor to AppOptics). Its API tokens can be used to access and manage monitoring data and configurations." +} diff --git a/pkg/detectors/solarwindsobservability/solarwindsobservability_integration_test.go b/pkg/detectors/solarwindsobservability/solarwindsobservability_integration_test.go new file mode 100644 index 000000000000..95952677870b --- /dev/null +++ b/pkg/detectors/solarwindsobservability/solarwindsobservability_integration_test.go @@ -0,0 +1,167 @@ +//go:build detectors +// +build detectors + +package solarwindsobservability + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestSolarWindsObservability_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("SOLARWINDSOBSERVABILITY") + inactiveSecret := testSecrets.MustGetField("SOLARWINDSOBSERVABILITY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a solarwinds secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SolarWindsObservability, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a solarwinds secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detector_typepb.DetectorType_SolarWindsObservability, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a solarwinds secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detector_typepb.DetectorType_SolarWindsObservability, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500 from region ap-01")) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a solarwinds secret %s within but not valid", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SolarWindsObservability, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("SolarWindsObservability.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "SecretParts", "ExtraData") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("SolarWindsObservability.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/appoptics/appoptics_test.go b/pkg/detectors/solarwindsobservability/solarwindsobservability_test.go similarity index 85% rename from pkg/detectors/appoptics/appoptics_test.go rename to pkg/detectors/solarwindsobservability/solarwindsobservability_test.go index a29bfd069432..5161d6e74258 100644 --- a/pkg/detectors/appoptics/appoptics_test.go +++ b/pkg/detectors/solarwindsobservability/solarwindsobservability_test.go @@ -1,4 +1,4 @@ -package appoptics +package solarwindsobservability import ( "context" @@ -11,7 +11,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAppOptics_Pattern(t *testing.T) { +func TestSolarWindsObservability_Pattern(t *testing.T) { d := Scanner{} ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) @@ -23,11 +23,11 @@ func TestAppOptics_Pattern(t *testing.T) { { name: "valid pattern", input: ` - func validateAppOpticsKey() bool { - appopticsKey := "Xwl4ViaAFDLrAmFX9g1blkUVC5dJj2he3a1tzkpJ4-PznQukQruRjqMFbEG73L92LJyBGMZ" + func validateSolarWindsKey() bool { + solarwindsKey := "Xwl4ViaAFDLrAmFX9g1blkUVC5dJj2he3a1tzkpJ4-PznQukQruRjqMFbEG73L92LJyBGMZ" log.Println("Checking API key status...") - if !isActive(appopticsKey) { + if !isActive(solarwindsKey) { log.Println("API key is inactive or invalid.") return false } @@ -42,7 +42,7 @@ func TestAppOptics_Pattern(t *testing.T) { input: ` GLOBAL - {appoptics} + {solarwinds} {AQAAABAAA zxsb8yzT0RbIJ1TAalB87LOVUcT1b4uEgvT4tXCcSqv_gcmlrx5aQRleHPDFKePjpHFof5J} configuration for production 2023-05-18T14:32:10Z @@ -54,11 +54,11 @@ func TestAppOptics_Pattern(t *testing.T) { { name: "invalid pattern", input: ` - func validateAppOpticsKey() bool { - appopticsKey := "Xwl4ViaAFDLrAmFX9g1blkUVC5dJj2h:3a1tzkpJ43PznQukQruRjqMFbEG73L92LJyBGMZ" + func validateSolarWindsKey() bool { + solarwindsKey := "Xwl4ViaAFDLrAmFX9g1blkUVC5dJj2h:3a1tzkpJ43PznQukQruRjqMFbEG73L92LJyBGMZ" log.Println("Checking API key status...") - if !isActive(appopticsKey) { + if !isActive(solarwindsKey) { log.Println("API key is inactive or invalid.") return false } diff --git a/pkg/detectors/sonarcloud/sonarcloud.go b/pkg/detectors/sonarcloud/v1/sonarcloud.go similarity index 95% rename from pkg/detectors/sonarcloud/sonarcloud.go rename to pkg/detectors/sonarcloud/v1/sonarcloud.go index 6e236f1a796d..c60595174fc7 100644 --- a/pkg/detectors/sonarcloud/sonarcloud.go +++ b/pkg/detectors/sonarcloud/v1/sonarcloud.go @@ -20,6 +20,9 @@ type Scanner struct { // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 1 } var ( defaultClient = common.SaneHttpClient() @@ -56,6 +59,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detector_typepb.DetectorType_SonarCloud, Raw: []byte(match), SecretParts: map[string]string{"key": match}, + ExtraData: map[string]string{ + "version": fmt.Sprintf("%d", s.Version()), + }, } if verify { diff --git a/pkg/detectors/sonarcloud/sonarcloud_integration_test.go b/pkg/detectors/sonarcloud/v1/sonarcloud_integration_test.go similarity index 91% rename from pkg/detectors/sonarcloud/sonarcloud_integration_test.go rename to pkg/detectors/sonarcloud/v1/sonarcloud_integration_test.go index ced576bcf216..5e41fbac6e90 100644 --- a/pkg/detectors/sonarcloud/sonarcloud_integration_test.go +++ b/pkg/detectors/sonarcloud/v1/sonarcloud_integration_test.go @@ -19,7 +19,7 @@ import ( func TestSonarCloud_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } @@ -50,6 +50,12 @@ func TestSonarCloud_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_SonarCloud, Verified: true, + ExtraData: map[string]string{ + "version": "1", + }, + SecretParts: map[string]string{ + "key": secret, + }, }, }, wantErr: false, @@ -66,6 +72,12 @@ func TestSonarCloud_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_SonarCloud, Verified: false, + ExtraData: map[string]string{ + "version": "1", + }, + SecretParts: map[string]string{ + "key": inactiveSecret, + }, }, }, wantErr: false, diff --git a/pkg/detectors/sonarcloud/sonarcloud_test.go b/pkg/detectors/sonarcloud/v1/sonarcloud_test.go similarity index 100% rename from pkg/detectors/sonarcloud/sonarcloud_test.go rename to pkg/detectors/sonarcloud/v1/sonarcloud_test.go diff --git a/pkg/detectors/sonarcloud/v2/sonarcloud.go b/pkg/detectors/sonarcloud/v2/sonarcloud.go new file mode 100644 index 000000000000..be3fd4732ec9 --- /dev/null +++ b/pkg/detectors/sonarcloud/v2/sonarcloud.go @@ -0,0 +1,124 @@ +package sonarcloud + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +func (Scanner) Version() int { return 2 } + +var ( + defaultClient = common.SaneHttpClient() + + // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. + keyPat = regexp.MustCompile(`\b(sqco_[[:alnum:]]{59})\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"sqco_"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// FromData will find and optionally verify SonarCloud secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + uniqueTokenMatches := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokenMatches[match[1]] = struct{}{} + } + + for match := range uniqueTokenMatches { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_SonarCloud, + Raw: []byte(match), + SecretParts: map[string]string{"key": match}, + ExtraData: map[string]string{ + "version": fmt.Sprintf("%d", s.Version()), + }, + } + + if verify { + isVerified, verificationErr := s.verifyMatch(ctx, s.getClient(), match) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, match) + } + + results = append(results, s1) + } + + return results, nil +} + +// verifyMatch attempts to validate a SonarCloud token. +func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) { + // Format-compliant (lowercase alphanumeric + hyphens) dummy key that won't match a + // real organization, so the API returns 404 rather than rejecting it as malformed. + organizationKey := "nonexistent-org-9f2a7c4e1b8d4a6f" + url := fmt.Sprintf("https://sonarcloud.io/api/projects/search?organization=%s", organizationKey) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if err != nil { + return false, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("failed to perform request: %w", err) + } + + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + // The official Sonar API docs don't document the response codes for this + // endpoint, but community evidence indicates 403 is returned when the + // token is valid but lacks org-admin permissions on the queried org: + // https://community.sonarsource.com/t/api-access-issue-for-read-only-user-despite-ui-access/140025 + // https://community.sonarsource.com/t/http-403-when-accessing-api-projects-search-but-other-endpoint-works-like-components/20668 + switch res.StatusCode { + case http.StatusOK, http.StatusNotFound, http.StatusForbidden: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_SonarCloud +} + +func (s Scanner) Description() string { + return "SonarCloud is a cloud-based code quality and security service. SonarCloud tokens can be used to access project analysis and quality reports." +} diff --git a/pkg/detectors/sonarcloud/v2/sonarcloud_integration_test.go b/pkg/detectors/sonarcloud/v2/sonarcloud_integration_test.go new file mode 100644 index 000000000000..409c58f29ff3 --- /dev/null +++ b/pkg/detectors/sonarcloud/v2/sonarcloud_integration_test.go @@ -0,0 +1,132 @@ +//go:build detectors +// +build detectors + +package sonarcloud + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestSonarCloud_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("SONARCLOUD_V2") + inactiveSecret := testSecrets.MustGetField("SONARCLOUD_V2_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SonarCloud, + Verified: true, + ExtraData: map[string]string{ + "version": "2", + }, + SecretParts: map[string]string{ + "key": secret, + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SonarCloud, + Verified: false, + ExtraData: map[string]string{ + "version": "2", + }, + SecretParts: map[string]string{ + "key": inactiveSecret, + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("SonarCloud.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("SonarCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/sonarcloud/v2/sonarcloud_test.go b/pkg/detectors/sonarcloud/v2/sonarcloud_test.go new file mode 100644 index 000000000000..a901276bd6a7 --- /dev/null +++ b/pkg/detectors/sonarcloud/v2/sonarcloud_test.go @@ -0,0 +1,85 @@ +package sonarcloud + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "sqco_FbT1v5HrMX6qyKUkaoLQdYOniAVhbEccERraQABuhBayDOkyCZTa8TDQFRp" + invalidPattern = "sqco_FbT1v5HrMX6qyKUkaoLQdYOniAVhbEccERraQABuhBayDOkyCZTa8TDQF==" +) + +func TestSonarCloud_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("token = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern - ignore duplicate", + input: fmt.Sprintf("token = '%s' | '%s'", validPattern, validPattern), + want: []string{validPattern}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("token = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/sourcegraph/sourcegraph.go b/pkg/detectors/sourcegraph/sourcegraph.go index 53b1987c466c..ee2fd254425d 100644 --- a/pkg/detectors/sourcegraph/sourcegraph.go +++ b/pkg/detectors/sourcegraph/sourcegraph.go @@ -44,6 +44,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Sourcegraph, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } s1.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/sourcegraph/", @@ -65,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true @@ -78,7 +79,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } else { s1.SetVerificationError(err, resMatch) } - s1.SecretParts = map[string]string{"key": resMatch} } results = append(results, s1) diff --git a/pkg/detectors/sourcegraphcody/sourcegraphcody.go b/pkg/detectors/sourcegraphcody/sourcegraphcody.go index b26882d05a9a..304e0aebc291 100644 --- a/pkg/detectors/sourcegraphcody/sourcegraphcody.go +++ b/pkg/detectors/sourcegraphcody/sourcegraphcody.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 403 || res.StatusCode == 401 { diff --git a/pkg/detectors/sparkpost/sparkpost.go b/pkg/detectors/sparkpost/sparkpost.go index d5e39e1fd02e..b6e2f8fe3188 100644 --- a/pkg/detectors/sparkpost/sparkpost.go +++ b/pkg/detectors/sparkpost/sparkpost.go @@ -2,10 +2,11 @@ package sparkpost import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/spectralops/spectralops.go b/pkg/detectors/spectralops/spectralops.go new file mode 100644 index 000000000000..4133a9b8f333 --- /dev/null +++ b/pkg/detectors/spectralops/spectralops.go @@ -0,0 +1,131 @@ +package spectralops + +import ( + "context" + "fmt" + "io" + "net/http" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Compile-time interface check +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + + // SpectralOps Personal API keys: + // Format: spu- + 32 lowercase aplhabets or digits + spectralTokenPat = regexp.MustCompile( + `\b(spu-[a-z0-9]{32})\b`, + ) +) + +// Keywords used for fast pre-filtering +func (s Scanner) Keywords() []string { + return []string{"spu-"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + +// FromData scans for SpectralOps API tokens and optionally verifies them +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + + dataStr := string(data) + + uniqueTokens := make(map[string]struct{}) + for _, match := range spectralTokenPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[match[1]] = struct{}{} + } + + for token := range uniqueTokens { + result := detectors.Result{ + DetectorType: detector_typepb.DetectorType_SpectralOps, + Raw: []byte(token), + SecretParts: map[string]string{ + "token": token, + }, + } + + if verify { + verified, verificationErr := verifySpectralToken( + ctx, + s.getClient(), + token, + ) + result.SetVerificationError(verificationErr, token) + result.Verified = verified + } + + results = append(results, result) + } + + return +} + +func verifySpectralToken( + ctx context.Context, + client *http.Client, + token string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://get.spectralops.io/api/v1/users", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Bearer "+token) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + // Token is invalid or revoked + return false, nil + default: + return false, fmt.Errorf( + "unexpected HTTP response status %d", + res.StatusCode, + ) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_SpectralOps +} + +func (s Scanner) Description() string { + return "SpectralOps is a DevSecOps platform for detecting secrets and misconfigurations. Spectral personal API tokens can be used to access and interact with Spectral services and resources." +} diff --git a/pkg/detectors/spectralops/spectralops_integration_test.go b/pkg/detectors/spectralops/spectralops_integration_test.go new file mode 100644 index 000000000000..a6608693beaf --- /dev/null +++ b/pkg/detectors/spectralops/spectralops_integration_test.go @@ -0,0 +1,175 @@ +//go:build detectors +// +build detectors + +package spectralops + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestSpectralOps_FromData(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // Secrets are expected to be stored similarly to other detector tests + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + activeToken := testSecrets.MustGetField("SPECTRALOPS_PERSONAL_TOKEN") + inactiveToken := "spu-3f10194ca38240ddb880bab79492384b" + + type args struct { + ctx context.Context + data []byte + verify bool + } + + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Spectral API token %s for scan", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SpectralOps, + Verified: true, + Raw: []byte(activeToken), + }, + }, + }, + { + name: "found, real token, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Spectral API token %s for scan", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SpectralOps, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, real token, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Spectral API token %s for scan", activeToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SpectralOps, + Verified: false, + Raw: []byte(activeToken), + }, + }, + wantVerificationErr: true, + }, + { + name: "found, unverified (inactive token)", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: fmt.Appendf([]byte{}, "Using Spectral API token %s for scan", inactiveToken), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_SpectralOps, + Verified: false, + Raw: []byte(inactiveToken), + }, + }, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("no secrets here"), + verify: true, + }, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Fatalf("SpectralOps.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf( + "wantVerificationError = %v, verification error = %v", + tt.wantVerificationErr, + got[i].VerificationError(), + ) + } + } + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("SpectralOps.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkSpectralOps_FromData(b *testing.B) { + ctx := context.Background() + s := Scanner{} + + for name, data := range detectors.MustGetBenchmarkData() { + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/spectralops/spectralops_test.go b/pkg/detectors/spectralops/spectralops_test.go new file mode 100644 index 000000000000..1701fb227af6 --- /dev/null +++ b/pkg/detectors/spectralops/spectralops_test.go @@ -0,0 +1,128 @@ +package spectralops + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestSpectralOps_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern - basic", + input: ` + [INFO] Running spectral scan + [DEBUG] Using token spu-3f10194ca38240ddb880bab79492384b + [INFO] Scan complete + `, + want: []string{ + "spu-3f10194ca38240ddb880bab79492384b", + }, + }, + { + name: "valid pattern - with keyword spectral nearby", + input: ` + [INFO] spectral initialized + [DEBUG] SPECTRAL_API_KEY=spu-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + `, + want: []string{ + "spu-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }, + { + name: "valid pattern - multiple tokens", + input: ` + spu-11111111111111111111111111111111 + spu-22222222222222222222222222222222 + `, + want: []string{ + "spu-11111111111111111111111111111111", + "spu-22222222222222222222222222222222", + }, + }, + { + name: "invalid pattern - uppercase characters", + input: ` + [DEBUG] token=spu-3F10194CA38240DDB880BAB79492384B + `, + want: nil, + }, + { + name: "invalid pattern - too short", + input: ` + [DEBUG] token=spu-1234 + `, + want: nil, + }, + { + name: "invalid pattern - invalid token length", + input: ` + [DEBUG] token=spu-3f10194ca38240ddb880bab79492384badwad + `, + want: nil, + }, + { + name: "invalid pattern - keyword only", + input: ` + [INFO] spectral scan started spu- + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found in the input", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + if len(results) != len(test.want) { + t.Errorf( + "mismatch in result count: expected %d, got %d", + len(test.want), + len(results), + ) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/speechtextai/speechtextai.go b/pkg/detectors/speechtextai/speechtextai.go index c090483c5ac8..71419880c5c6 100644 --- a/pkg/detectors/speechtextai/speechtextai.go +++ b/pkg/detectors/speechtextai/speechtextai.go @@ -3,10 +3,11 @@ package speechtextai import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken.go index a15becbaed77..3c6b13da1d8c 100644 --- a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken.go +++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("X-Sf-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/spoonacular/spoonacular.go b/pkg/detectors/spoonacular/spoonacular.go index ee383d620145..45867a66d325 100644 --- a/pkg/detectors/spoonacular/spoonacular.go +++ b/pkg/detectors/spoonacular/spoonacular.go @@ -3,10 +3,11 @@ package spoonacular import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sportsmonk/sportsmonk.go b/pkg/detectors/sportsmonk/sportsmonk.go index 63fe85e6686e..5d8894e5296e 100644 --- a/pkg/detectors/sportsmonk/sportsmonk.go +++ b/pkg/detectors/sportsmonk/sportsmonk.go @@ -2,10 +2,11 @@ package sportsmonk import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sqlserver/sqlserver.go b/pkg/detectors/sqlserver/sqlserver.go index 220974d8f48e..5466578f1914 100644 --- a/pkg/detectors/sqlserver/sqlserver.go +++ b/pkg/detectors/sqlserver/sqlserver.go @@ -138,7 +138,7 @@ var ping = func(ctx context.Context, config msdsn.Config) (bool, error) { if err != nil { return false, err } - defer tcpConn.Close() + defer func() { _ = tcpConn.Close() }() cleanConfig := msdsn.Config{} cleanConfig.Host = config.Host diff --git a/pkg/detectors/square/square.go b/pkg/detectors/square/square.go index 536191ba25ec..638a9814a125 100644 --- a/pkg/detectors/square/square.go +++ b/pkg/detectors/square/square.go @@ -2,7 +2,9 @@ package square import ( "context" + "encoding/json" "fmt" + "io" "net/http" "strings" @@ -14,14 +16,21 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( + defaultClient = common.SaneHttpClient() + + // there are a few endpoints we can check, but merchants seems the least sensitive. + verifyURL = "https://connect.squareupsandbox.com/v2/merchants" + // more context to be added if this is too generic - secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"square"}) + `(EAAA[a-zA-Z0-9\-\+\=]{60})`) + secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"square"}) + `(EAAA[a-zA-Z0-9\-_+=]{60})`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -46,38 +55,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result result := detectors.Result{ DetectorType: detector_typepb.DetectorType_Square, Raw: []byte(resMatch), + SecretParts: map[string]string{"key": resMatch}, } result.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/square/", } if verify { - // there are a few endpoints we can check, but templates seems the least sensitive. - // 403 will be issued if the scope is wrong but the key is correct - baseURL := "https://connect.squareupsandbox.com/v2/merchants" - - client := common.SaneHttpClient() - - // test `merchants` scope - its commonly allowed and low sensitivity - req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - req.Header.Add("Content-Type", "application/json") - // unclear if this version needs to be set or matters, seems to work without, but docs want it - // req.Header.Add("Square-Version", "2020-08-12") - res, err := client.Do(req) - if err == nil { - res.Body.Close() // The request body is unused. - - // 200 means good key and has `merchants` scope - default allowed by square - // 401 is bad key - if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { - result.Verified = true - } + client := s.client + if client == nil { + client = defaultClient } - result.SecretParts = map[string]string{"key": resMatch} + + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + result.Verified = isVerified + result.SetVerificationError(verificationErr, resMatch) } results = append(results, result) @@ -86,6 +78,89 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return } +func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, verifyURL, nil) + if err != nil { + return false, err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Add("Content-Type", "application/json") + // unclear if this version needs to be set or matters, seems to work without, but docs want it + // req.Header.Add("Square-Version", "2020-08-12") + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + // good key and has `merchants` scope - default allowed by square + return true, nil + case http.StatusUnauthorized: + return false, nil + case http.StatusForbidden: + // A 403 only proves the key is valid when Square itself says the scope is insufficient. + // Anything else may come from a TLS-intercepting proxy or Square's edge (rate limiting, bot protection). + body := readBody(res) + if hasErrorCode(body, "INSUFFICIENT_SCOPES") { + return true, nil + } + return false, fmt.Errorf("unexpected 403 from Square: %s", truncateBody(body)) + default: + return false, fmt.Errorf("unexpected HTTP response status %d: %s", res.StatusCode, truncateBody(readBody(res))) + } +} + +// squareErrorResponse is Square's error envelope for non-2xx responses. +// See https://developer.squareup.com/docs/build-basics/handling-errors. +type squareErrorResponse struct { + Errors []struct { + Category string `json:"category"` + Code string `json:"code"` + Detail string `json:"detail"` + } `json:"errors"` +} + +const ( + maxBodySize = 4 << 10 // cap how much of the response body we read + maxErrorBodySize = 512 // cap how much of the body we put in an error +) + +func readBody(res *http.Response) []byte { + body, _ := io.ReadAll(io.LimitReader(res.Body, maxBodySize)) + return body +} + +func hasErrorCode(body []byte, code string) bool { + var apiErr squareErrorResponse + if err := json.Unmarshal(body, &apiErr); err != nil { + return false + } + for _, e := range apiErr.Errors { + if e.Code == code { + return true + } + } + return false +} + +// truncateBody collapses whitespace and caps the body so an HTML block page doesn't flood the output. +func truncateBody(body []byte) string { + s := strings.Join(strings.Fields(string(body)), " ") + if s == "" { + return "" + } + if len(s) > maxErrorBodySize { + return s[:maxErrorBodySize] + "..." + } + return s +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Square } diff --git a/pkg/detectors/square/square_test.go b/pkg/detectors/square/square_test.go index dadda930f3ea..c04b1dd68cba 100644 --- a/pkg/detectors/square/square_test.go +++ b/pkg/detectors/square/square_test.go @@ -3,10 +3,13 @@ package square import ( "context" "fmt" + "net/http" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -89,3 +92,108 @@ func TestSquare_Pattern(t *testing.T) { }) } } + +func TestSquare_Verification(t *testing.T) { + input := fmt.Sprintf("%s token = '%s'", keyword, validPattern) + + tests := []struct { + name string + status int + body string + wantVerified bool + wantErr bool + wantErrParts []string + }{ + { + name: "200 is verified", + status: http.StatusOK, + body: `{"merchant":[]}`, + wantVerified: true, + }, + { + name: "401 is unverified", + status: http.StatusUnauthorized, + body: `{"errors":[{"category":"AUTHENTICATION_ERROR","code":"UNAUTHORIZED","detail":"This request could not be authorized."}]}`, + }, + { + name: "403 with INSUFFICIENT_SCOPES is verified", + status: http.StatusForbidden, + body: `{"errors":[{"category":"AUTHENTICATION_ERROR","code":"INSUFFICIENT_SCOPES","detail":"The merchant has not given your application sufficient permissions."}]}`, + wantVerified: true, + }, + { + name: "403 with HTML body is indeterminate", + status: http.StatusForbidden, + body: "\n Blocked by corporate proxy\n", + wantErr: true, + wantErrParts: []string{"403", " Blocked by corporate proxy "}, + }, + { + name: "403 with other Square code is indeterminate", + status: http.StatusForbidden, + body: `{"errors":[{"category":"AUTHENTICATION_ERROR","code":"FORBIDDEN","detail":"Forbidden"}]}`, + wantErr: true, + wantErrParts: []string{"403", "FORBIDDEN"}, + }, + { + name: "403 with empty body is indeterminate", + status: http.StatusForbidden, + wantErr: true, + wantErrParts: []string{"403", ""}, + }, + { + name: "429 is indeterminate", + status: http.StatusTooManyRequests, + body: `{"errors":[{"category":"RATE_LIMIT_ERROR","code":"RATE_LIMITED"}]}`, + wantErr: true, + wantErrParts: []string{"429", "RATE_LIMITED"}, + }, + { + name: "500 with large body is truncated", + status: http.StatusInternalServerError, + body: strings.Repeat("x", 10000), + wantErr: true, + wantErrParts: []string{"500", strings.Repeat("x", maxErrorBodySize) + "..."}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := Scanner{client: common.ConstantResponseHttpClient(test.status, test.body)} + + results, err := s.FromData(context.Background(), true, []byte(input)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + r := results[0] + if r.Verified != test.wantVerified { + t.Errorf("Verified = %v, want %v", r.Verified, test.wantVerified) + } + + verificationErr := r.VerificationError() + if (verificationErr != nil) != test.wantErr { + t.Fatalf("VerificationError = %v, wantErr %v", verificationErr, test.wantErr) + } + if verificationErr == nil { + return + } + + msg := verificationErr.Error() + for _, part := range test.wantErrParts { + if !strings.Contains(msg, part) { + t.Errorf("VerificationError %q does not contain %q", msg, part) + } + } + if strings.Contains(msg, validPattern) { + t.Errorf("VerificationError leaks the token: %q", msg) + } + if len(msg) > maxErrorBodySize+64 { + t.Errorf("VerificationError not truncated: %d bytes", len(msg)) + } + }) + } +} diff --git a/pkg/detectors/squarespace/squarespace.go b/pkg/detectors/squarespace/squarespace.go index bad86c3c5d5a..aa37fe2ab33c 100644 --- a/pkg/detectors/squarespace/squarespace.go +++ b/pkg/detectors/squarespace/squarespace.go @@ -3,10 +3,11 @@ package squarespace import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sslmate/sslmate.go b/pkg/detectors/sslmate/sslmate.go index ea931dd9853e..1075b5e0580b 100644 --- a/pkg/detectors/sslmate/sslmate.go +++ b/pkg/detectors/sslmate/sslmate.go @@ -3,10 +3,11 @@ package sslmate import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/statuscake/statuscake.go b/pkg/detectors/statuscake/statuscake.go index 0eb8f395ef0d..a65c314ed7ec 100644 --- a/pkg/detectors/statuscake/statuscake.go +++ b/pkg/detectors/statuscake/statuscake.go @@ -3,10 +3,11 @@ package statuscake import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/statuspage/statuspage.go b/pkg/detectors/statuspage/statuspage.go index bab989d218ac..df02537e257b 100644 --- a/pkg/detectors/statuspage/statuspage.go +++ b/pkg/detectors/statuspage/statuspage.go @@ -2,10 +2,11 @@ package statuspage import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/statuspal/statuspal.go b/pkg/detectors/statuspal/statuspal.go index a7ab099baacd..85d3b92a0d7f 100644 --- a/pkg/detectors/statuspal/statuspal.go +++ b/pkg/detectors/statuspal/statuspal.go @@ -2,10 +2,11 @@ package statuspal import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/stitchdata/stitchdata.go b/pkg/detectors/stitchdata/stitchdata.go index dfd5a3f99900..fb1814236630 100644 --- a/pkg/detectors/stitchdata/stitchdata.go +++ b/pkg/detectors/stitchdata/stitchdata.go @@ -3,10 +3,11 @@ package stitchdata import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/stockdata/stockdata.go b/pkg/detectors/stockdata/stockdata.go index d8de59edf306..da836d9259fe 100644 --- a/pkg/detectors/stockdata/stockdata.go +++ b/pkg/detectors/stockdata/stockdata.go @@ -2,10 +2,11 @@ package stockdata import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/storecove/storecove.go b/pkg/detectors/storecove/storecove.go index a019b323a6ed..2fc7ea01c8df 100644 --- a/pkg/detectors/storecove/storecove.go +++ b/pkg/detectors/storecove/storecove.go @@ -3,10 +3,11 @@ package storecove import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/stormboard/stormboard.go b/pkg/detectors/stormboard/stormboard.go index e7b2ede18f49..dc8d40e2850f 100644 --- a/pkg/detectors/stormboard/stormboard.go +++ b/pkg/detectors/stormboard/stormboard.go @@ -2,10 +2,11 @@ package stormboard import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/stormglass/stormglass.go b/pkg/detectors/stormglass/stormglass.go index 7bad5e5cc181..3bdf9809db76 100644 --- a/pkg/detectors/stormglass/stormglass.go +++ b/pkg/detectors/stormglass/stormglass.go @@ -2,10 +2,11 @@ package stormglass import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/storychief/storychief.go b/pkg/detectors/storychief/storychief.go index 94c371d7c3b5..57dbe62db194 100644 --- a/pkg/detectors/storychief/storychief.go +++ b/pkg/detectors/storychief/storychief.go @@ -3,10 +3,11 @@ package storychief import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/strava/strava.go b/pkg/detectors/strava/strava.go index bcb11cc8676a..aecf2691116f 100644 --- a/pkg/detectors/strava/strava.go +++ b/pkg/detectors/strava/strava.go @@ -2,10 +2,11 @@ package strava import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -70,7 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/streak/streak.go b/pkg/detectors/streak/streak.go index 4d16eb95818a..f4fe12a6a341 100644 --- a/pkg/detectors/streak/streak.go +++ b/pkg/detectors/streak/streak.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/stripe/stripe.go b/pkg/detectors/stripe/stripe.go index 807f5fa5e5be..ba1b6b895575 100644 --- a/pkg/detectors/stripe/stripe.go +++ b/pkg/detectors/stripe/stripe.go @@ -40,6 +40,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result result := detectors.Result{ DetectorType: detector_typepb.DetectorType_Stripe, Raw: []byte(match), + SecretParts: map[string]string{"key": match}, } result.ExtraData = map[string]string{ "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/", @@ -60,13 +61,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - res.Body.Close() // The request body is unused. + _ = res.Body.Close() // The request body is unused. if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { result.Verified = true } } - result.SecretParts = map[string]string{"key": match} } results = append(results, result) diff --git a/pkg/detectors/stripo/stripo.go b/pkg/detectors/stripo/stripo.go index 02e42976e27a..d500b2c42c12 100644 --- a/pkg/detectors/stripo/stripo.go +++ b/pkg/detectors/stripo/stripo.go @@ -3,10 +3,11 @@ package stripo import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Stripo-Api-Auth", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/stytch/stytch.go b/pkg/detectors/stytch/stytch.go index 00c25fb6fd93..e5aa7decdbd0 100644 --- a/pkg/detectors/stytch/stytch.go +++ b/pkg/detectors/stytch/stytch.go @@ -2,10 +2,11 @@ package stytch import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/sugester/sugester.go b/pkg/detectors/sugester/sugester.go index 8f75b4463a0b..9fd9a611d8ab 100644 --- a/pkg/detectors/sugester/sugester.go +++ b/pkg/detectors/sugester/sugester.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Accept", "application/vnd.sugester+json; version=3") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/supabasetoken/supabasetoken.go b/pkg/detectors/supabasetoken/supabasetoken.go index 35c54bf292cb..3e42704fd1bd 100644 --- a/pkg/detectors/supabasetoken/supabasetoken.go +++ b/pkg/detectors/supabasetoken/supabasetoken.go @@ -3,10 +3,11 @@ package supabasetoken import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/supernotesapi/supernotesapi.go b/pkg/detectors/supernotesapi/supernotesapi.go index de4410245132..7dfed4e501e8 100644 --- a/pkg/detectors/supernotesapi/supernotesapi.go +++ b/pkg/detectors/supernotesapi/supernotesapi.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/surveyanyplace/surveyanyplace.go b/pkg/detectors/surveyanyplace/surveyanyplace.go index f24eaa544b75..7705f2eaa5e2 100644 --- a/pkg/detectors/surveyanyplace/surveyanyplace.go +++ b/pkg/detectors/surveyanyplace/surveyanyplace.go @@ -3,10 +3,11 @@ package surveyanyplace import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -67,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/surveybot/surveybot.go b/pkg/detectors/surveybot/surveybot.go index 232722d0a7ce..8f828d9a4e1a 100644 --- a/pkg/detectors/surveybot/surveybot.go +++ b/pkg/detectors/surveybot/surveybot.go @@ -3,10 +3,11 @@ package surveybot import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Api-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { var r response if err := json.NewDecoder(res.Body).Decode(&r); err != nil { diff --git a/pkg/detectors/surveysparrow/surveysparrow.go b/pkg/detectors/surveysparrow/surveysparrow.go index 1edd6f0133b5..67ed2496fd90 100644 --- a/pkg/detectors/surveysparrow/surveysparrow.go +++ b/pkg/detectors/surveysparrow/surveysparrow.go @@ -3,10 +3,11 @@ package surveysparrow import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/survicate/survicate.go b/pkg/detectors/survicate/survicate.go index 4003418523c8..5fa3e85ae415 100644 --- a/pkg/detectors/survicate/survicate.go +++ b/pkg/detectors/survicate/survicate.go @@ -3,10 +3,11 @@ package survicate import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/swell/swell.go b/pkg/detectors/swell/swell.go index 585e41579a8c..d7ffeffab64b 100644 --- a/pkg/detectors/swell/swell.go +++ b/pkg/detectors/swell/swell.go @@ -2,10 +2,11 @@ package swell import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/swiftype/swiftype.go b/pkg/detectors/swiftype/swiftype.go index acd67dab0774..dd74df69bc0e 100644 --- a/pkg/detectors/swiftype/swiftype.go +++ b/pkg/detectors/swiftype/swiftype.go @@ -2,10 +2,11 @@ package swiftype import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tableau/tableau.go b/pkg/detectors/tableau/tableau.go index e0370aca019b..a6c163e71df7 100644 --- a/pkg/detectors/tableau/tableau.go +++ b/pkg/detectors/tableau/tableau.go @@ -89,6 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result Raw: []byte(tokenName), RawV2: []byte(fmt.Sprintf("%s:%s:%s", tokenName, tokenSecret, endpoint)), ExtraData: make(map[string]string), + SecretParts: map[string]string{"token_name": tokenName, "pat_secret": tokenSecret, "endpoint": endpoint}, } if verify { @@ -97,9 +98,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result result.Verified = isVerified maps.Copy(result.ExtraData, extraData) result.SetVerificationError(verificationErr, tokenName, tokenSecret, endpoint) - if isVerified { - result.SecretParts = map[string]string{"token_name": tokenName, "pat_secret": tokenSecret, "endpoint": endpoint} - } } results = append(results, result) } diff --git a/pkg/detectors/tailscale/tailscale.go b/pkg/detectors/tailscale/tailscale.go index e155c5312730..696400afa760 100644 --- a/pkg/detectors/tailscale/tailscale.go +++ b/pkg/detectors/tailscale/tailscale.go @@ -3,11 +3,12 @@ package tailscale import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "net/url" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Content-Type", "application/x-www-form-urlencoded") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusNoContent: s1.Verified = true diff --git a/pkg/detectors/tallyfy/tallyfy.go b/pkg/detectors/tallyfy/tallyfy.go index cfb003bae761..401a9cacc641 100644 --- a/pkg/detectors/tallyfy/tallyfy.go +++ b/pkg/detectors/tallyfy/tallyfy.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tatumio/tatumio.go b/pkg/detectors/tatumio/tatumio.go index 9c0494b45474..7fac9fc5854b 100644 --- a/pkg/detectors/tatumio/tatumio.go +++ b/pkg/detectors/tatumio/tatumio.go @@ -2,10 +2,11 @@ package tatumio import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-api-key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/taxjar/taxjar.go b/pkg/detectors/taxjar/taxjar.go index 3707612a9fc9..08ba1f81d955 100644 --- a/pkg/detectors/taxjar/taxjar.go +++ b/pkg/detectors/taxjar/taxjar.go @@ -3,10 +3,11 @@ package taxjar import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/teamgate/teamgate.go b/pkg/detectors/teamgate/teamgate.go index dc87d52b79ac..36f95166641b 100644 --- a/pkg/detectors/teamgate/teamgate.go +++ b/pkg/detectors/teamgate/teamgate.go @@ -2,10 +2,11 @@ package teamgate import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -61,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/teamworkcrm/teamworkcrm.go b/pkg/detectors/teamworkcrm/teamworkcrm.go index 6f5387d46e1c..5f651bc67814 100644 --- a/pkg/detectors/teamworkcrm/teamworkcrm.go +++ b/pkg/detectors/teamworkcrm/teamworkcrm.go @@ -3,10 +3,11 @@ package teamworkcrm import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/teamworkdesk/teamworkdesk.go b/pkg/detectors/teamworkdesk/teamworkdesk.go index 507171c299b6..dc96b4360541 100644 --- a/pkg/detectors/teamworkdesk/teamworkdesk.go +++ b/pkg/detectors/teamworkdesk/teamworkdesk.go @@ -3,10 +3,11 @@ package teamworkdesk import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/teamworkspaces/teamworkspaces.go b/pkg/detectors/teamworkspaces/teamworkspaces.go index c2fcd8d2aaa5..cf77e426aedb 100644 --- a/pkg/detectors/teamworkspaces/teamworkspaces.go +++ b/pkg/detectors/teamworkspaces/teamworkspaces.go @@ -3,10 +3,11 @@ package teamworkspaces import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/technicalanalysisapi/technicalanalysisapi.go b/pkg/detectors/technicalanalysisapi/technicalanalysisapi.go index 2ec3dbdf183f..1f9bbadbafd5 100644 --- a/pkg/detectors/technicalanalysisapi/technicalanalysisapi.go +++ b/pkg/detectors/technicalanalysisapi/technicalanalysisapi.go @@ -3,11 +3,12 @@ package technicalanalysisapi import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tefter/tefter.go b/pkg/detectors/tefter/tefter.go index 93eba4b0b26f..90153847dab3 100644 --- a/pkg/detectors/tefter/tefter.go +++ b/pkg/detectors/tefter/tefter.go @@ -3,11 +3,12 @@ package tefter import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -47,25 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "POST", "https://www.tefter.io/api/bookmarks?url=google.com", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("X-User-Token", resMatch) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - validResponse := json.Valid(bodyBytes) - - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -74,6 +59,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "POST", "https://www.tefter.io/api/bookmarks?url=google.com", nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-User-Token", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + validResponse := json.Valid(bodyBytes) + + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Tefter } diff --git a/pkg/detectors/telegrambottoken/telegrambottoken.go b/pkg/detectors/telegrambottoken/telegrambottoken.go index 6010fadbcd13..f68b17fb5b00 100644 --- a/pkg/detectors/telegrambottoken/telegrambottoken.go +++ b/pkg/detectors/telegrambottoken/telegrambottoken.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true diff --git a/pkg/detectors/teletype/teletype.go b/pkg/detectors/teletype/teletype.go index 7368e6516825..792d040c05d9 100644 --- a/pkg/detectors/teletype/teletype.go +++ b/pkg/detectors/teletype/teletype.go @@ -2,11 +2,12 @@ package teletype import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,31 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.teletype.app/public/api/v1/messages", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("X-Auth-Token", resMatch) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `"code":401`) - - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = false - } else { - s1.Verified = true - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -79,6 +58,38 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.teletype.app/public/api/v1/messages", nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-Auth-Token", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `"code":401`) + + if res.StatusCode >= 200 && res.StatusCode < 300 { + if !validResponse { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Teletype } diff --git a/pkg/detectors/telnyx/telnyx.go b/pkg/detectors/telnyx/telnyx.go index d6af0965c523..b2ed2bd9e0c7 100644 --- a/pkg/detectors/telnyx/telnyx.go +++ b/pkg/detectors/telnyx/telnyx.go @@ -3,10 +3,11 @@ package telnyx import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken.go b/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken.go index 5c1adf13ccb1..d72af7032df1 100644 --- a/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken.go +++ b/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/textmagic/textmagic.go b/pkg/detectors/textmagic/textmagic.go index 3efefa83442e..c81b68dc0dee 100644 --- a/pkg/detectors/textmagic/textmagic.go +++ b/pkg/detectors/textmagic/textmagic.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -68,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/theoddsapi/theoddsapi.go b/pkg/detectors/theoddsapi/theoddsapi.go index 60ee52ec40c9..80ee5d994562 100644 --- a/pkg/detectors/theoddsapi/theoddsapi.go +++ b/pkg/detectors/theoddsapi/theoddsapi.go @@ -2,10 +2,11 @@ package theoddsapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.go index bdbcbcb3420f..4c549c8e00ad 100644 --- a/pkg/detectors/thinkific/thinkific.go +++ b/pkg/detectors/thinkific/thinkific.go @@ -52,27 +52,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - domainRes := fmt.Sprintf("%s-s-school", resDomainMatch) - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.thinkific.com/api/public/v1/collections", nil) - if err != nil { - continue - } - req.Header.Add("X-Auth-API-Key", resMatch) - req.Header.Add("X-Auth-Subdomain", domainRes) - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - if strings.Contains(body, "API Access is not available") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resDomainMatch, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resDomainMatch, resMatch) } results = append(results, s1) @@ -82,6 +64,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resDomainMatch string, resMatch string) (bool, error) { + domainRes := fmt.Sprintf("%s-s-school", resDomainMatch) + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.thinkific.com/api/public/v1/collections", nil) + if err != nil { + return false, err + } + req.Header.Add("X-Auth-API-Key", resMatch) + req.Header.Add("X-Auth-Subdomain", domainRes) + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + if strings.Contains(body, "API Access is not available") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Thinkific } diff --git a/pkg/detectors/thousandeyes/thousandeyes.go b/pkg/detectors/thousandeyes/thousandeyes.go index 7bc3df09ccb9..b246247cdb6f 100644 --- a/pkg/detectors/thousandeyes/thousandeyes.go +++ b/pkg/detectors/thousandeyes/thousandeyes.go @@ -2,10 +2,11 @@ package thousandeyes import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.SetBasicAuth(userPatMatch, tokenPatMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ticketmaster/ticketmaster.go b/pkg/detectors/ticketmaster/ticketmaster.go index e7c68dcce448..b097ac36f017 100644 --- a/pkg/detectors/ticketmaster/ticketmaster.go +++ b/pkg/detectors/ticketmaster/ticketmaster.go @@ -3,10 +3,11 @@ package ticketmaster import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tiingo/tiingo.go b/pkg/detectors/tiingo/tiingo.go index 3a1e124ea8cc..3bc121876d05 100644 --- a/pkg/detectors/tiingo/tiingo.go +++ b/pkg/detectors/tiingo/tiingo.go @@ -3,10 +3,11 @@ package tiingo import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/timecamp/timecamp.go b/pkg/detectors/timecamp/timecamp.go index 168b6e3331ba..51f7fd1d5f5e 100644 --- a/pkg/detectors/timecamp/timecamp.go +++ b/pkg/detectors/timecamp/timecamp.go @@ -2,10 +2,11 @@ package timecamp import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/timezoneapi/timezoneapi.go b/pkg/detectors/timezoneapi/timezoneapi.go index a1433612c4d5..5b243e87aed8 100644 --- a/pkg/detectors/timezoneapi/timezoneapi.go +++ b/pkg/detectors/timezoneapi/timezoneapi.go @@ -2,10 +2,11 @@ package timezoneapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { return nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse { s1.Verified = true } diff --git a/pkg/detectors/tineswebhook/tineswebhook.go b/pkg/detectors/tineswebhook/tineswebhook.go index 55415b2ccc37..333db9d9e79a 100644 --- a/pkg/detectors/tineswebhook/tineswebhook.go +++ b/pkg/detectors/tineswebhook/tineswebhook.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tly/tly.go b/pkg/detectors/tly/tly.go index 8e38eafe9f26..8ca17235d553 100644 --- a/pkg/detectors/tly/tly.go +++ b/pkg/detectors/tly/tly.go @@ -2,10 +2,13 @@ package tly import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" + "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -30,38 +33,76 @@ func (s Scanner) Keywords() []string { } // FromData will find and optionally verify TLy secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + dataStr := string(data) - for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) + uniqueKeys := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueKeys[strings.TrimSpace(match[1])] = struct{}{} + } - s1 := detectors.Result{ + for key := range uniqueKeys { + result := detectors.Result{ DetectorType: detector_typepb.DetectorType_TLy, - Raw: []byte(resMatch), - SecretParts: map[string]string{"key": resMatch}, + Raw: []byte(key), + SecretParts: map[string]string{ + "key": key, + }, } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://t.ly/api/v1/link/stats?api_token="+resMatch+"&short_url=https://t.ly/h9YS", nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + verified, verificationErr := verifyTLyKey(ctx, client, key) + result.SetVerificationError(verificationErr, key) + result.Verified = verified } - results = append(results, s1) + results = append(results, result) } - return results, nil + return +} + +func verifyTLyKey( + ctx context.Context, + client *http.Client, + key string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://api.t.ly/api/v1/link/list", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Bearer "+key) + req.Header.Set("Accept", "application/json") + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/tly/tly_integration_test.go b/pkg/detectors/tly/tly_integration_test.go index 502170071344..65e831ef267d 100644 --- a/pkg/detectors/tly/tly_integration_test.go +++ b/pkg/detectors/tly/tly_integration_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -96,8 +96,19 @@ func TestTLy_FromChunk(t *testing.T) { } got[i].Raw = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("TLy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Tly.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) } diff --git a/pkg/detectors/tmetric/tmetric.go b/pkg/detectors/tmetric/tmetric.go index c9d6e8b0ebb0..703ed4b82b7d 100644 --- a/pkg/detectors/tmetric/tmetric.go +++ b/pkg/detectors/tmetric/tmetric.go @@ -3,10 +3,11 @@ package tmetric import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/todoist/todoist.go b/pkg/detectors/todoist/todoist.go index 6181f9d72aa1..612ad84103e6 100644 --- a/pkg/detectors/todoist/todoist.go +++ b/pkg/detectors/todoist/todoist.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/toggltrack/toggltrack.go b/pkg/detectors/toggltrack/toggltrack.go index 909533407484..5c267a344a96 100644 --- a/pkg/detectors/toggltrack/toggltrack.go +++ b/pkg/detectors/toggltrack/toggltrack.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_TogglTrack, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tokeet/tokeet.go b/pkg/detectors/tokeet/tokeet.go index e0823f10a3f0..f658716db1a0 100644 --- a/pkg/detectors/tokeet/tokeet.go +++ b/pkg/detectors/tokeet/tokeet.go @@ -3,10 +3,11 @@ package tokeet import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tomorrowio/tomorrowio.go b/pkg/detectors/tomorrowio/tomorrowio.go index 4a9011ac87c9..6252d21313aa 100644 --- a/pkg/detectors/tomorrowio/tomorrowio.go +++ b/pkg/detectors/tomorrowio/tomorrowio.go @@ -3,10 +3,11 @@ package tomorrowio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tomtom/tomtom.go b/pkg/detectors/tomtom/tomtom.go index 007f1eeb07d6..e3085e26b4c5 100644 --- a/pkg/detectors/tomtom/tomtom.go +++ b/pkg/detectors/tomtom/tomtom.go @@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tradier/tradier.go b/pkg/detectors/tradier/tradier.go index dedd679e9bb6..0bb66bb35381 100644 --- a/pkg/detectors/tradier/tradier.go +++ b/pkg/detectors/tradier/tradier.go @@ -3,10 +3,11 @@ package tradier import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/transferwise/transferwise.go b/pkg/detectors/transferwise/transferwise.go index 1fa16c9e837c..d2d1ae2e7033 100644 --- a/pkg/detectors/transferwise/transferwise.go +++ b/pkg/detectors/transferwise/transferwise.go @@ -3,10 +3,11 @@ package transferwise import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/transport_audit_test.go b/pkg/detectors/transport_audit_test.go index 8e3f9f9d6df2..378e3f827d93 100644 --- a/pkg/detectors/transport_audit_test.go +++ b/pkg/detectors/transport_audit_test.go @@ -3,9 +3,10 @@ package detectors_test import ( "os" "path/filepath" - "regexp" "strings" "testing" + + regexp "github.com/wasilibs/go-re2" ) // bypassPatterns are stdlib HTTP entry points that bypass the shared diff --git a/pkg/detectors/travelpayouts/travelpayouts.go b/pkg/detectors/travelpayouts/travelpayouts.go index aaee81115cd9..a663ba0dbfc7 100644 --- a/pkg/detectors/travelpayouts/travelpayouts.go +++ b/pkg/detectors/travelpayouts/travelpayouts.go @@ -3,10 +3,11 @@ package travelpayouts import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/travisci/travisci.go b/pkg/detectors/travisci/travisci.go index bcfbbd033adc..5597a45b096b 100644 --- a/pkg/detectors/travisci/travisci.go +++ b/pkg/detectors/travisci/travisci.go @@ -3,10 +3,11 @@ package travisci import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Travis-API-Version", "3") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/trelloapikey/trelloapikey.go b/pkg/detectors/trelloapikey/trelloapikey.go index ed6962486310..76ef96d4def4 100644 --- a/pkg/detectors/trelloapikey/trelloapikey.go +++ b/pkg/detectors/trelloapikey/trelloapikey.go @@ -2,10 +2,11 @@ package trelloapikey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/tru/tru.go b/pkg/detectors/tru/tru.go index 3ba432c3566d..45eb53b4a371 100644 --- a/pkg/detectors/tru/tru.go +++ b/pkg/detectors/tru/tru.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -68,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", baseToken)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go b/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go index d0415b3c8995..248336e1fe4f 100644 --- a/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go +++ b/pkg/detectors/trufflehogenterprise/trufflehogenterprise.go @@ -3,10 +3,11 @@ package trufflehogenterprise import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -75,7 +76,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse { s1.Verified = true diff --git a/pkg/detectors/twelvedata/twelvedata.go b/pkg/detectors/twelvedata/twelvedata.go index 5f779eea7e7e..c16add1364bb 100644 --- a/pkg/detectors/twelvedata/twelvedata.go +++ b/pkg/detectors/twelvedata/twelvedata.go @@ -2,11 +2,12 @@ package twelvedata import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,29 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.twelvedata.com/earliest_timestamp?symbol=AAPL&interval=1day&apikey="+resMatch, nil) - if err != nil { - continue - } - - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} - // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} - // ingenious! - - if !strings.Contains(body, "401") { - s1.Verified = true - } - - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -77,6 +58,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.twelvedata.com/earliest_timestamp?symbol=AAPL&interval=1day&apikey="+resMatch, nil) + if err != nil { + return false, err + } + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} + // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} + // ingenious! + + if !strings.Contains(body, "401") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_TwelveData } diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.go index aeaa09682b68..ef10f53f0a34 100644 --- a/pkg/detectors/twilio/twilio.go +++ b/pkg/detectors/twilio/twilio.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "maps" "net/http" regexp "github.com/wasilibs/go-re2" @@ -23,7 +24,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.RetryableHTTPClient() + defaultClient = detectors.NewClientWithDedup(common.RetryableHTTPClient()) sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`) keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`) ) @@ -56,16 +57,23 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - keyMatches := keyPat.FindAllString(dataStr, -1) - sidMatches := sidPat.FindAllString(dataStr, -1) + uniqueKeys := make(map[string]struct{}) + for _, k := range keyPat.FindAllString(dataStr, -1) { + uniqueKeys[k] = struct{}{} + } + uniqueSIDs := make(map[string]struct{}) + for _, s := range sidPat.FindAllString(dataStr, -1) { + uniqueSIDs[s] = struct{}{} + } - for _, sid := range sidMatches { - for _, key := range keyMatches { + for sid := range uniqueSIDs { + for key := range uniqueKeys { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Twilio, Raw: []byte(sid), RawV2: []byte(sid + key), Redacted: sid, + SecretParts: map[string]string{"key": key, "sid": sid}, } s1.ExtraData = map[string]string{ @@ -77,13 +85,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.SetVerificationError(verificationErr) - for key, value := range extraData { - s1.ExtraData[key] = value - } - - if s1.Verified { - s1.SecretParts = map[string]string{"key": key, "sid": sid} - } + maps.Copy(s1.ExtraData, extraData) } results = append(results, s1) @@ -104,15 +106,15 @@ func (s Scanner) Description() string { func verifyTwilio(ctx context.Context, client *http.Client, key, sid string) (map[string]string, bool, error) { req, err := http.NewRequestWithContext(ctx, "GET", "https://verify.twilio.com/v2/Services", nil) if err != nil { - return nil, false, nil + return nil, false, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Accept", "*/*") req.SetBasicAuth(sid, key) - resp, err := client.Do(req) + resp, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Twilio, sid+key, req) if err != nil { - return nil, false, nil + return nil, false, err } defer func() { _, _ = io.Copy(io.Discard, resp.Body) diff --git a/pkg/detectors/twilio/twilio_test.go b/pkg/detectors/twilio/twilio_test.go index a7f30df60f67..2e772dba3ac4 100644 --- a/pkg/detectors/twilio/twilio_test.go +++ b/pkg/detectors/twilio/twilio_test.go @@ -2,11 +2,19 @@ package twilio import ( "context" + "errors" "fmt" + "io" + "net/http" + "strings" + "sync/atomic" "testing" + "time" "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-retryablehttp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -81,3 +89,118 @@ func TestTwilio_Pattern(t *testing.T) { }) } } + +type errorTransport struct{ err error } + +func (t errorTransport) RoundTrip(*http.Request) (*http.Response, error) { return nil, t.err } + +func TestTwilio_VerificationDeterminacy(t *testing.T) { + data := []byte(fmt.Sprintf("twilio sid %s key %s", validSid, validKey)) + + tests := []struct { + name string + client *http.Client + wantVerified bool + wantVerificationErr bool + }{ + { + name: "authenticated rejection is determinate", + client: common.ConstantResponseHttpClient(http.StatusUnauthorized, ""), + wantVerified: false, + wantVerificationErr: false, + }, + { + name: "success is determinate", + client: common.ConstantResponseHttpClient(http.StatusOK, `{"services":[{"friendly_name":"n","sid":"s","account_sid":"a"}]}`), + wantVerified: true, + wantVerificationErr: false, + }, + { + name: "connection reset is indeterminate", + client: &http.Client{Transport: errorTransport{err: errors.New("read: connection reset by peer")}}, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "timeout is indeterminate", + client: &http.Client{Transport: errorTransport{err: context.DeadlineExceeded}}, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "rate limit reaching the switch is indeterminate", + client: common.ConstantResponseHttpClient(http.StatusTooManyRequests, ""), + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "server error is indeterminate", + client: common.ConstantResponseHttpClient(http.StatusInternalServerError, "{}"), + wantVerified: false, + wantVerificationErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + results, err := Scanner{client: test.client}.FromData(context.Background(), true, data) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + got := results[0] + if got.Verified != test.wantVerified { + t.Errorf("Verified = %v, want %v", got.Verified, test.wantVerified) + } + if (got.VerificationError() != nil) != test.wantVerificationErr { + t.Errorf("VerificationError() = %v, want error presence %v", + got.VerificationError(), test.wantVerificationErr) + } + if e := got.VerificationError(); e != nil && strings.Contains(e.Error(), validKey) { + t.Errorf("verification error leaks the credential: %v", e) + } + }) + } +} + +func TestTwilio_RetryExhaustionIsIndeterminate(t *testing.T) { + var attempts int32 + + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 3 + retryClient.Logger = nil + retryClient.RetryWaitMin = time.Millisecond + retryClient.RetryWaitMax = 2 * time.Millisecond + retryClient.HTTPClient.Transport = common.FakeTransport{ + CreateResponse: func(req *http.Request) (*http.Response, error) { + atomic.AddInt32(&attempts, 1) + return &http.Response{ + Request: req, + StatusCode: http.StatusTooManyRequests, + Body: io.NopCloser(strings.NewReader("")), + }, nil + }, + } + + data := []byte(fmt.Sprintf("twilio sid %s key %s", validSid, validKey)) + results, err := Scanner{client: retryClient.StandardClient()}.FromData(context.Background(), true, data) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + if got := atomic.LoadInt32(&attempts); got != 4 { + t.Errorf("expected 4 attempts (initial + RetryMax), got %d", got) + } + if results[0].Verified { + t.Error("Verified = true, want false") + } + if results[0].VerificationError() == nil { + t.Error("VerificationError() = nil; a throttled credential would be recorded as rotated") + } +} diff --git a/pkg/detectors/twilioapikey/twilioapikey.go b/pkg/detectors/twilioapikey/twilioapikey.go index e686062be009..83bb07d7f5de 100644 --- a/pkg/detectors/twilioapikey/twilioapikey.go +++ b/pkg/detectors/twilioapikey/twilioapikey.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "maps" "net/http" regexp "github.com/wasilibs/go-re2" @@ -23,7 +24,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient()) apiKeyPat = regexp.MustCompile(`\bSK[a-zA-Z0-9]{32}\b`) secretPat = regexp.MustCompile(`\b[0-9a-zA-Z]{32}\b`) ) @@ -54,17 +55,24 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - apiKeyMatches := apiKeyPat.FindAllString(dataStr, -1) - secretMatches := secretPat.FindAllString(dataStr, -1) + uniqueAPIKeys := make(map[string]struct{}) + for _, k := range apiKeyPat.FindAllString(dataStr, -1) { + uniqueAPIKeys[k] = struct{}{} + } + uniqueSecrets := make(map[string]struct{}) + for _, s := range secretPat.FindAllString(dataStr, -1) { + uniqueSecrets[s] = struct{}{} + } - for _, apiKey := range apiKeyMatches { - for _, secret := range secretMatches { + for apiKey := range uniqueAPIKeys { + for secret := range uniqueSecrets { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Twilio, Raw: []byte(apiKey), RawV2: []byte(apiKey + secret), Redacted: secret[:5] + "...", ExtraData: make(map[string]string), + SecretParts: map[string]string{"key": apiKey, "sid": secret}, } if verify { @@ -72,13 +80,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = isVerified s1.SetVerificationError(verificationErr) - for key, value := range extraData { - s1.ExtraData[key] = value - } - - if s1.Verified { - s1.SecretParts = map[string]string{"key": apiKey, "sid": secret} - } + maps.Copy(s1.ExtraData, extraData) } results = append(results, s1) @@ -99,16 +101,16 @@ func (s Scanner) Description() string { func verifyTwilioAPIKey(ctx context.Context, client *http.Client, apiKey, secret string) (map[string]string, bool, error) { req, err := http.NewRequestWithContext(ctx, "GET", "https://verify.twilio.com/v2/Services", nil) if err != nil { - return nil, false, nil + return nil, false, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Accept", "*/*") req.SetBasicAuth(apiKey, secret) - resp, err := client.Do(req) + resp, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_TwilioApiKey, apiKey+secret, req) if err != nil { - return nil, false, nil + return nil, false, err } defer func() { _, _ = io.Copy(io.Discard, resp.Body) diff --git a/pkg/detectors/twilioapikey/twilioapikey_test.go b/pkg/detectors/twilioapikey/twilioapikey_test.go index 4e61f3462788..3046ba42177c 100644 --- a/pkg/detectors/twilioapikey/twilioapikey_test.go +++ b/pkg/detectors/twilioapikey/twilioapikey_test.go @@ -2,11 +2,15 @@ package twilioapikey import ( "context" + "errors" "fmt" + "net/http" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) @@ -102,3 +106,79 @@ func TestTwilioAPIKey_SecretRedacted(t *testing.T) { t.Errorf("expected redacted secret to be '%s', got '%s'", validSecret[:5]+"...", results[0].Redacted) } } + +type errorTransport struct{ err error } + +func (t errorTransport) RoundTrip(*http.Request) (*http.Response, error) { return nil, t.err } + +func TestTwilioAPIKey_VerificationDeterminacy(t *testing.T) { + data := []byte(fmt.Sprintf("twilio %s %s", validAPIKey, validSecret)) + + tests := []struct { + name string + client *http.Client + wantVerified bool + wantVerificationErr bool + }{ + { + name: "authenticated rejection is determinate", + client: common.ConstantResponseHttpClient(http.StatusUnauthorized, ""), + wantVerified: false, + wantVerificationErr: false, + }, + { + name: "success is determinate", + client: common.ConstantResponseHttpClient(http.StatusOK, `{"services":[{"friendly_name":"n","sid":"s","account_sid":"a"}]}`), + wantVerified: true, + wantVerificationErr: false, + }, + { + name: "rate limit is indeterminate", + client: common.ConstantResponseHttpClient(http.StatusTooManyRequests, ""), + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "connection reset is indeterminate", + client: &http.Client{Transport: errorTransport{err: errors.New("read: connection reset by peer")}}, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "timeout is indeterminate", + client: &http.Client{Transport: errorTransport{err: context.DeadlineExceeded}}, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "server error is indeterminate", + client: common.ConstantResponseHttpClient(http.StatusInternalServerError, "{}"), + wantVerified: false, + wantVerificationErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + results, err := Scanner{client: test.client}.FromData(context.Background(), true, data) + if err != nil { + t.Fatalf("FromData() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + got := results[0] + if got.Verified != test.wantVerified { + t.Errorf("Verified = %v, want %v", got.Verified, test.wantVerified) + } + if (got.VerificationError() != nil) != test.wantVerificationErr { + t.Errorf("VerificationError() = %v, want error presence %v", + got.VerificationError(), test.wantVerificationErr) + } + if e := got.VerificationError(); e != nil && strings.Contains(e.Error(), validSecret) { + t.Errorf("verification error leaks the credential: %v", e) + } + }) + } +} diff --git a/pkg/detectors/twist/twist.go b/pkg/detectors/twist/twist.go index c59f90cf8526..24d4f20db8fc 100644 --- a/pkg/detectors/twist/twist.go +++ b/pkg/detectors/twist/twist.go @@ -3,10 +3,11 @@ package twist import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", setAuth)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/twitch/twitch.go b/pkg/detectors/twitch/twitch.go index c5276b9b4706..e6ef01968fcb 100644 --- a/pkg/detectors/twitch/twitch.go +++ b/pkg/detectors/twitch/twitch.go @@ -108,7 +108,7 @@ func verifyTwitch(ctx context.Context, client *http.Client, resMatch string, res if err != nil { return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: diff --git a/pkg/detectors/twitter/v1/twitter_v1.go b/pkg/detectors/twitter/v1/twitter_v1.go index 094288f185a1..12f686b2c2b5 100644 --- a/pkg/detectors/twitter/v1/twitter_v1.go +++ b/pkg/detectors/twitter/v1/twitter_v1.go @@ -94,7 +94,7 @@ func (s Scanner) VerifyTwitterToken(ctx context.Context, client *http.Client, to return false, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: return true, nil diff --git a/pkg/detectors/twitterconsumerkey/twitterconsumerkey.go b/pkg/detectors/twitterconsumerkey/twitterconsumerkey.go index 3a6687b9be6d..0de1d7af3d75 100644 --- a/pkg/detectors/twitterconsumerkey/twitterconsumerkey.go +++ b/pkg/detectors/twitterconsumerkey/twitterconsumerkey.go @@ -5,6 +5,7 @@ import ( b64 "encoding/base64" "encoding/json" "fmt" + "io" "net/http" "strings" @@ -103,7 +104,7 @@ func verifyBearerToken(ctx context.Context, client *http.Client, token string) ( req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK, http.StatusForbidden: // 403 indicates lack of permission, but valid token (could be due to twitter free tier) @@ -132,7 +133,7 @@ func fetchBearerToken(ctx context.Context, client *http.Client, key, secret stri if err != nil { return "", err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() switch res.StatusCode { case http.StatusOK: @@ -142,6 +143,13 @@ func fetchBearerToken(ctx context.Context, client *http.Client, key, secret stri } return token.AccessToken, nil default: + body, readErr := io.ReadAll(res.Body) + if readErr != nil { + return "", fmt.Errorf("unexpected HTTP response status %d; failed to read response body: %w", res.StatusCode, readErr) + } + if len(body) > 0 { + return "", fmt.Errorf("unexpected HTTP response status %d: %s", res.StatusCode, strings.TrimSpace(string(body))) + } return "", fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } } diff --git a/pkg/detectors/tyntec/tyntec.go b/pkg/detectors/tyntec/tyntec.go index 692fa4907ee3..fc81b1d28bb0 100644 --- a/pkg/detectors/tyntec/tyntec.go +++ b/pkg/detectors/tyntec/tyntec.go @@ -2,10 +2,11 @@ package tyntec import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("apiKey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/typeform/v2/typeform.go b/pkg/detectors/typeform/v2/typeform.go index 89885dc0888e..d44bd65bf124 100644 --- a/pkg/detectors/typeform/v2/typeform.go +++ b/pkg/detectors/typeform/v2/typeform.go @@ -100,16 +100,17 @@ func verifyMatch(ctx context.Context, client *http.Client, secret string) (bool, _ = res.Body.Close() }() - if res.StatusCode == 200 { + switch res.StatusCode { + case 200: var response *TypeFormResponse if err = json.NewDecoder(res.Body).Decode(&response); err != nil { return false, nil, err } return true, response, nil - } else if res.StatusCode == 401 || res.StatusCode == 403 { + case 401, 403: return false, nil, nil - } else { + default: return false, nil, fmt.Errorf("unexpected status code %d", res.StatusCode) } } diff --git a/pkg/detectors/typetalk/typetalk.go b/pkg/detectors/typetalk/typetalk.go index c27764be4d80..d7938aac97d8 100644 --- a/pkg/detectors/typetalk/typetalk.go +++ b/pkg/detectors/typetalk/typetalk.go @@ -3,10 +3,11 @@ package typetalk import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -59,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/ubidots/ubidots.go b/pkg/detectors/ubidots/ubidots.go index b18ea4e18798..a6f7e28e4f81 100644 --- a/pkg/detectors/ubidots/ubidots.go +++ b/pkg/detectors/ubidots/ubidots.go @@ -2,10 +2,11 @@ package ubidots import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Auth-Token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/uclassify/uclassify.go b/pkg/detectors/uclassify/uclassify.go index c97189fe7b92..21710289d838 100644 --- a/pkg/detectors/uclassify/uclassify.go +++ b/pkg/detectors/uclassify/uclassify.go @@ -3,10 +3,11 @@ package uclassify import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/unifyid/unifyid.go b/pkg/detectors/unifyid/unifyid.go index 25a4b2ad5dd7..be01f3e1d67b 100644 --- a/pkg/detectors/unifyid/unifyid.go +++ b/pkg/detectors/unifyid/unifyid.go @@ -2,11 +2,12 @@ package unifyid import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,25 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - payload := strings.NewReader(`{"token": "sample"}`) - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.unify.id/v1/humandetect/verify", payload) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("X-API-Key", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "invalid token")) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -73,6 +58,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + payload := strings.NewReader(`{"token": "sample"}`) + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.unify.id/v1/humandetect/verify", payload) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-API-Key", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "invalid token")) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_UnifyID } diff --git a/pkg/detectors/unplugg/unplugg.go b/pkg/detectors/unplugg/unplugg.go index 981d9381f0b8..1d90993a1fd1 100644 --- a/pkg/detectors/unplugg/unplugg.go +++ b/pkg/detectors/unplugg/unplugg.go @@ -2,10 +2,11 @@ package unplugg import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("x-access-token", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/unsplash/unsplash.go b/pkg/detectors/unsplash/unsplash.go index fce5168664b9..7638ab3c245a 100644 --- a/pkg/detectors/unsplash/unsplash.go +++ b/pkg/detectors/unsplash/unsplash.go @@ -2,10 +2,11 @@ package unsplash import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/upcdatabase/upcdatabase.go b/pkg/detectors/upcdatabase/upcdatabase.go index b0227be30d56..b6f60110a556 100644 --- a/pkg/detectors/upcdatabase/upcdatabase.go +++ b/pkg/detectors/upcdatabase/upcdatabase.go @@ -3,11 +3,12 @@ package upcdatabase import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `added_time`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/uplead/uplead.go b/pkg/detectors/uplead/uplead.go index 536300baa040..da41156eb488 100644 --- a/pkg/detectors/uplead/uplead.go +++ b/pkg/detectors/uplead/uplead.go @@ -2,10 +2,11 @@ package uplead import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/uploadcare/uploadcare.go b/pkg/detectors/uploadcare/uploadcare.go index 12ba6f78e1f4..d05452fae381 100644 --- a/pkg/detectors/uploadcare/uploadcare.go +++ b/pkg/detectors/uploadcare/uploadcare.go @@ -3,10 +3,11 @@ package uploadcare import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Uploadcare.Simple %s:%s", publicKeyMatch, resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/uptimerobot/uptimerobot.go b/pkg/detectors/uptimerobot/uptimerobot.go index 18c372f8a1f7..e41dcc97ade4 100644 --- a/pkg/detectors/uptimerobot/uptimerobot.go +++ b/pkg/detectors/uptimerobot/uptimerobot.go @@ -47,27 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.uptimerobot.com/v2/getMonitors?api_key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `"ok"`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = true - } else { - s1.Verified = false - } - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -76,6 +58,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.uptimerobot.com/v2/getMonitors?api_key="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `"ok"`) + if res.StatusCode >= 200 && res.StatusCode < 300 { + if validResponse { + return true, nil + } + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_UptimeRobot } diff --git a/pkg/detectors/upwave/upwave.go b/pkg/detectors/upwave/upwave.go index 95d22f6505d6..ecf542db48ce 100644 --- a/pkg/detectors/upwave/upwave.go +++ b/pkg/detectors/upwave/upwave.go @@ -3,10 +3,11 @@ package upwave import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index e36ccdf56d77..a04ffd13167d 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -18,6 +18,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) +var errNoSuchHost = errors.New("no such host") + type Scanner struct { allowKnownTestSites bool client *http.Client @@ -112,21 +114,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if verify { hostname := parsedURL.Hostname() if hostNotFoundCache.Exists(hostname) { - logger.V(3).Info("Skipping uri: no such host", "host", hostname) - continue - } - - if s.client == nil { - s.client = defaultClient - } - isVerified, vErr := verifyURL(ctx, s.client, parsedURL) - r.Verified = isVerified - if vErr != nil { - var dnsErr *net.DNSError - if errors.As(vErr, &dnsErr) && dnsErr.IsNotFound { - hostNotFoundCache.Set(hostname, struct{}{}) + // An earlier candidate already proved this host does not resolve, so the lookup is skipped. + // The finding is still reported: dropping it here means the reported secrets depend both on + // whether verification was requested and on the order candidates happened to be processed, + // since the first candidate for the host was reported and later ones were not. + logger.V(3).Info("Skipping uri verification: no such host", "host", hostname) + r.SetVerificationError(errNoSuchHost, password) + } else { + if s.client == nil { + s.client = defaultClient + } + isVerified, vErr := verifyURL(ctx, s.client, parsedURL) + r.Verified = isVerified + if vErr != nil { + var dnsErr *net.DNSError + if errors.As(vErr, &dnsErr) && dnsErr.IsNotFound { + hostNotFoundCache.Set(hostname, struct{}{}) + } + r.SetVerificationError(vErr, password) } - r.SetVerificationError(vErr, password) } } diff --git a/pkg/detectors/urlscan/urlscan.go b/pkg/detectors/urlscan/urlscan.go index 2291df6afde9..05ec485ff107 100644 --- a/pkg/detectors/urlscan/urlscan.go +++ b/pkg/detectors/urlscan/urlscan.go @@ -2,10 +2,11 @@ package urlscan import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("API-Key", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/user/user.go b/pkg/detectors/user/user.go index 50c2242264f7..d0ec714ec351 100644 --- a/pkg/detectors/user/user.go +++ b/pkg/detectors/user/user.go @@ -2,71 +2,140 @@ package user import ( "context" + "errors" "fmt" "net/http" + "net/url" "strings" regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter +} var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + _ detectors.EndpointCustomizer = (*Scanner)(nil) + + defaultClient = detectors.NewClientWithDedup(detectors.DetectorHttpClientWithNoLocalAddresses) + + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([A-Za-z0-9]{64})\b`) + userURLPat = regexp.MustCompile(`\b([a-z0-9-]+\.user\.com)\b`) - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"user"}) + `\b([a-zA-Z0-9-._+=]{64})\b`) + invalidHosts = simple.NewCache[struct{}]() + errNoHost = errors.New("no such host") ) +func (Scanner) CloudEndpoint() string { return "" } + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"user"} + return []string{"user.com"} +} + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient } // FromData will find and optionally verify User secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + var uniqueTokens, uniqueURLs = make(map[string]struct{}), make(map[string]struct{}) - for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueTokens[strings.TrimSpace(match[1])] = struct{}{} + } - s1 := detectors.Result{ - DetectorType: detector_typepb.DetectorType_User, - Raw: []byte(resMatch), - SecretParts: map[string]string{"key": resMatch}, - } + var foundURLs = make([]string, 0) + for _, match := range userURLPat.FindAllStringSubmatch(dataStr, -1) { + foundURLs = append(foundURLs, match[1]) + } + + // Merge found endpoints with any user-configured endpoints. + // Callers must call UseFoundEndpoints(true) to include endpoints extracted from data. + for _, endpoint := range s.Endpoints(foundURLs...) { + endpoint = strings.TrimPrefix(endpoint, "https://") + uniqueURLs[endpoint] = struct{}{} + } - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://secretscanner.user.com/api/public/users/", nil) - if err != nil { + for token := range uniqueTokens { + for hostname := range uniqueURLs { + if invalidHosts.Exists(hostname) { + delete(uniqueURLs, hostname) continue } - req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true + + endpointURL := url.URL{Scheme: "https", Host: hostname} + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_User, + Raw: []byte(token), + RawV2: []byte(token + ":" + endpointURL.String()), + SecretParts: map[string]string{ + "key": token, + "endpoint": endpointURL.String(), + }, + } + + if verify { + isVerified, verificationErr := verifyUserToken(ctx, s.getClient(), hostname, token) + s1.Verified = isVerified + if verificationErr != nil { + if errors.Is(verificationErr, errNoHost) { + invalidHosts.Set(hostname, struct{}{}) + continue + } + s1.SetVerificationError(verificationErr, token) } } - } - results = append(results, s1) + results = append(results, s1) + } } return results, nil } +func verifyUserToken(ctx context.Context, client *http.Client, hostname, token string) (bool, error) { + u := url.URL{Scheme: "https", Host: hostname, Path: "/api/public/users/"} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) + if err != nil { + return false, err + } + req.Header.Set("Authorization", fmt.Sprintf("Token %s", token)) + + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_User, token+":"+hostname, req) + if err != nil { + if strings.Contains(err.Error(), "no such host") { + return false, errNoHost + } + return false, err + } + defer func() { _ = res.Body.Close() }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden, http.StatusGone: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_User } diff --git a/pkg/detectors/user/user_integration_test.go b/pkg/detectors/user/user_integration_test.go index b48a6ba4a880..420edbb632a4 100644 --- a/pkg/detectors/user/user_integration_test.go +++ b/pkg/detectors/user/user_integration_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -25,6 +27,7 @@ func TestUser_FromChunk(t *testing.T) { } secret := testSecrets.MustGetField("USER") inactiveSecret := testSecrets.MustGetField("USER_INACTIVE") + endpoint := testSecrets.MustGetField("USER_ENDPOINT") type args struct { ctx context.Context @@ -43,13 +46,19 @@ func TestUser_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a user secret %s within", secret)), + data: []byte(fmt.Sprintf("user token = %s\nurl = %s", secret, endpoint)), verify: true, }, want: []detectors.Result{ { DetectorType: detector_typepb.DetectorType_User, Verified: true, + Raw: []byte(secret), + RawV2: []byte(secret + ":" + endpoint), + SecretParts: map[string]string{ + "key": secret, + "endpoint": endpoint, + }, }, }, wantErr: false, @@ -59,13 +68,19 @@ func TestUser_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a user secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("user token = %s\nurl = %s", inactiveSecret, endpoint)), verify: true, }, want: []detectors.Result{ { DetectorType: detector_typepb.DetectorType_User, Verified: false, + Raw: []byte(inactiveSecret), + RawV2: []byte(inactiveSecret + ":" + endpoint), + SecretParts: map[string]string{ + "key": inactiveSecret, + "endpoint": endpoint, + }, }, }, wantErr: false, @@ -84,8 +99,9 @@ func TestUser_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + tt.s.UseFoundEndpoints(true) + + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("User.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,15 +110,62 @@ func TestUser_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("User.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "chunkOffset", + "chunkOffsetSet", + ) + if diff := cmp.Diff(tt.want, got, ignoreOpts); diff != "" { + t.Errorf("User.FromData() %s diff: (-want +got)\n%s", tt.name, diff) } }) } } +func TestUser_FromChunk_WithCustomEndpoint(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("USER") + endpoint := testSecrets.MustGetField("USER_ENDPOINT") + + s := Scanner{} + s.UseFoundEndpoints(true) + if err := s.SetConfiguredEndpoints(endpoint); err != nil { + t.Fatal("Error in setting configured endpoint") + } + + data := []byte(fmt.Sprintf("user token = %s", secret)) + + got, err := s.FromData(ctx, true, data) + + require.NoError(t, err, "unexpected error from FromData") + require.Greater(t, len(got), 0, "expected at least 1 result") + + expectedRawV2 := []byte(secret + ":" + endpoint) + if string(got[0].RawV2) != string(expectedRawV2) { + t.Errorf("User.FromData() rawV2 mismatch: got %s, want %s", string(got[0].RawV2), string(expectedRawV2)) + } +} + func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := Scanner{} diff --git a/pkg/detectors/user/user_test.go b/pkg/detectors/user/user_test.go index 1847ee565f75..1fbbcf37f559 100644 --- a/pkg/detectors/user/user_test.go +++ b/pkg/detectors/user/user_test.go @@ -12,9 +12,10 @@ import ( ) var ( - validPattern = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa7+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd" - invalidPattern = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa?+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd" - keyword = "user" + validKey = "9UgOsRfud4RTyBQpJQFOQiwNQcfeLGHH1DDoxhgzCvBmccmVQ7MYB0ai3LXGZNMf" + validURL = "https://testdetector.user.com" + invalidKey = "OmFxWjhZvCpOeMsgTJdZMas+dlUpr=fa?+.QOKKYvi7RKWyBeHtaLa7_rzMhLrRd" + keyword = "user.com" ) func TestUser_Pattern(t *testing.T) { @@ -26,23 +27,33 @@ func TestUser_Pattern(t *testing.T) { want []string }{ { - name: "valid pattern - with keyword user", - input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), - want: []string{validPattern}, + name: "valid pattern - key and url present", + input: fmt.Sprintf("user token = '%s'\nurl = %s", validKey, validURL), + want: []string{validKey + ":" + validURL}, }, { - name: "valid pattern - ignore duplicate", - input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), - want: []string{validPattern}, + name: "valid pattern - ignore duplicate keys", + input: fmt.Sprintf("user token = '%s' | '%s'\nurl = %s", validKey, validKey, validURL), + want: []string{validKey + ":" + validURL}, }, { - name: "valid pattern - key out of prefix range", - input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern), + name: "no result - key without url", + input: fmt.Sprintf("user token = '%s'", validKey), want: []string{}, }, { - name: "invalid pattern", - input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), + name: "no result - url without key", + input: fmt.Sprintf("%s url = %s", keyword, validURL), + want: []string{}, + }, + { + name: "no result - key out of prefix range", + input: fmt.Sprintf("user keyword is not close to the real key in the data\n = '%s'\nurl = %s", validKey, validURL), + want: []string{}, + }, + { + name: "no result - invalid key format", + input: fmt.Sprintf("user = '%s'\nurl = %s", invalidKey, validURL), want: []string{}, }, } @@ -50,12 +61,14 @@ func TestUser_Pattern(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { + if len(matchedDetectors) == 0 && len(test.want) > 0 { t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - results, err := d.FromData(context.Background(), false, []byte(test.input)) + s := Scanner{} + s.UseFoundEndpoints(true) + results, err := s.FromData(context.Background(), false, []byte(test.input)) if err != nil { t.Errorf("error = %v", err) return diff --git a/pkg/detectors/userflow/userflow.go b/pkg/detectors/userflow/userflow.go index 118ce97f6f51..7982fa5d4e33 100644 --- a/pkg/detectors/userflow/userflow.go +++ b/pkg/detectors/userflow/userflow.go @@ -3,10 +3,11 @@ package userflow import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/userstack/userstack.go b/pkg/detectors/userstack/userstack.go index 6d0a201a43d2..8132d716b0d9 100644 --- a/pkg/detectors/userstack/userstack.go +++ b/pkg/detectors/userstack/userstack.go @@ -3,11 +3,12 @@ package userstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) valid := strings.Contains(bodyString, `is_mobile_device`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if valid { s1.Verified = true } else { diff --git a/pkg/detectors/vagrantcloudpersonaltoken/vagrantcloudpersonaltoken.go b/pkg/detectors/vagrantcloudpersonaltoken/vagrantcloudpersonaltoken.go index 3dd2cf7e8d90..f38a71631e83 100644 --- a/pkg/detectors/vagrantcloudpersonaltoken/vagrantcloudpersonaltoken.go +++ b/pkg/detectors/vagrantcloudpersonaltoken/vagrantcloudpersonaltoken.go @@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/vatlayer/vatlayer.go b/pkg/detectors/vatlayer/vatlayer.go index 3f16298915bc..d0022ce5e951 100644 --- a/pkg/detectors/vatlayer/vatlayer.go +++ b/pkg/detectors/vatlayer/vatlayer.go @@ -3,11 +3,12 @@ package vatlayer import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -57,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `vat_number`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/vbout/vbout.go b/pkg/detectors/vbout/vbout.go index 477e2dee690f..0738b6686eac 100644 --- a/pkg/detectors/vbout/vbout.go +++ b/pkg/detectors/vbout/vbout.go @@ -3,10 +3,11 @@ package vbout import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/vercel/vercel.go b/pkg/detectors/vercel/vercel.go index 336ed1d4b718..fb443fed9f97 100644 --- a/pkg/detectors/vercel/vercel.go +++ b/pkg/detectors/vercel/vercel.go @@ -3,10 +3,11 @@ package vercel import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/verifier/verifier.go b/pkg/detectors/verifier/verifier.go index d13df44c1511..61218168ee51 100644 --- a/pkg/detectors/verifier/verifier.go +++ b/pkg/detectors/verifier/verifier.go @@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/verimail/verimail.go b/pkg/detectors/verimail/verimail.go index 24c2fc95891a..b1e7c79576fd 100644 --- a/pkg/detectors/verimail/verimail.go +++ b/pkg/detectors/verimail/verimail.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/veriphone/veriphone.go b/pkg/detectors/veriphone/veriphone.go index e3a3ca6e2dff..b8bf1b0ab8e5 100644 --- a/pkg/detectors/veriphone/veriphone.go +++ b/pkg/detectors/veriphone/veriphone.go @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Veriphone, resMatch, req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/versioneye/versioneye.go b/pkg/detectors/versioneye/versioneye.go index e32474089521..880fb7d8389c 100644 --- a/pkg/detectors/versioneye/versioneye.go +++ b/pkg/detectors/versioneye/versioneye.go @@ -2,10 +2,11 @@ package versioneye import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("apiKey", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/viewneo/viewneo.go b/pkg/detectors/viewneo/viewneo.go index eb3381567988..7916f16bf96a 100644 --- a/pkg/detectors/viewneo/viewneo.go +++ b/pkg/detectors/viewneo/viewneo.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/virustotal/virustotal.go b/pkg/detectors/virustotal/virustotal.go index 591d8de9f154..7937f380681a 100644 --- a/pkg/detectors/virustotal/virustotal.go +++ b/pkg/detectors/virustotal/virustotal.go @@ -2,10 +2,11 @@ package virustotal import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -64,7 +65,7 @@ func verifyToken(ctx context.Context, client *http.Client, token string) bool { if err != nil { return false } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode < 200 || res.StatusCode >= 300 { return false diff --git a/pkg/detectors/visualcrossing/visualcrossing.go b/pkg/detectors/visualcrossing/visualcrossing.go index 01eded31e824..24844e07cd2f 100644 --- a/pkg/detectors/visualcrossing/visualcrossing.go +++ b/pkg/detectors/visualcrossing/visualcrossing.go @@ -54,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/voiceflow/voiceflow.go b/pkg/detectors/voiceflow/voiceflow.go index 59ba768a0020..30d6d3462fef 100644 --- a/pkg/detectors/voiceflow/voiceflow.go +++ b/pkg/detectors/voiceflow/voiceflow.go @@ -67,11 +67,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - if res.StatusCode == http.StatusOK { + switch res.StatusCode { + case http.StatusOK: s1.Verified = true - } else if res.StatusCode == http.StatusUnauthorized { + case http.StatusUnauthorized: // The secret is determinately not verified (nothing to do) - } else { + default: var buf bytes.Buffer var bodyString string _, err = io.Copy(&buf, res.Body) diff --git a/pkg/detectors/voicegain/voicegain.go b/pkg/detectors/voicegain/voicegain.go index 538b103b64cc..efa81366d12a 100644 --- a/pkg/detectors/voicegain/voicegain.go +++ b/pkg/detectors/voicegain/voicegain.go @@ -3,10 +3,11 @@ package voicegain import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/voodoosms/voodoosms.go b/pkg/detectors/voodoosms/voodoosms.go index 5fe86f6d969a..ce0dfe6979ae 100644 --- a/pkg/detectors/voodoosms/voodoosms.go +++ b/pkg/detectors/voodoosms/voodoosms.go @@ -3,10 +3,11 @@ package voodoosms import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/vouchery/vouchery.go b/pkg/detectors/vouchery/vouchery.go index cf2f3369c171..1cb4324fd3d6 100644 --- a/pkg/detectors/vouchery/vouchery.go +++ b/pkg/detectors/vouchery/vouchery.go @@ -3,10 +3,11 @@ package vouchery import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -65,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/vpnapi/vpnapi.go b/pkg/detectors/vpnapi/vpnapi.go index 9cc630afa009..29138432a887 100644 --- a/pkg/detectors/vpnapi/vpnapi.go +++ b/pkg/detectors/vpnapi/vpnapi.go @@ -2,10 +2,11 @@ package vpnapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/vultrapikey/vultrapikey.go b/pkg/detectors/vultrapikey/vultrapikey.go index d3210ec93875..7beca19b9e81 100644 --- a/pkg/detectors/vultrapikey/vultrapikey.go +++ b/pkg/detectors/vultrapikey/vultrapikey.go @@ -3,10 +3,11 @@ package vultrapikey import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/vyte/vyte.go b/pkg/detectors/vyte/vyte.go index 5127a9ebb780..c166a1c31613 100644 --- a/pkg/detectors/vyte/vyte.go +++ b/pkg/detectors/vyte/vyte.go @@ -2,10 +2,11 @@ package vyte import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", resMatch) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/walkscore/walkscore.go b/pkg/detectors/walkscore/walkscore.go index 3b6f26951c28..c622108f389c 100644 --- a/pkg/detectors/walkscore/walkscore.go +++ b/pkg/detectors/walkscore/walkscore.go @@ -3,11 +3,12 @@ package walkscore import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -47,23 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://transit.walkscore.com/transit/search/stops/?lat=47.6101359&lon=-122.3420567&wsapikey=%s", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, `distance`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -72,6 +59,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://transit.walkscore.com/transit/search/stops/?lat=47.6101359&lon=-122.3420567&wsapikey=%s", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, `distance`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_WalkScore } diff --git a/pkg/detectors/weatherbit/weatherbit.go b/pkg/detectors/weatherbit/weatherbit.go index 02b7e6b29f6f..4c07720e9042 100644 --- a/pkg/detectors/weatherbit/weatherbit.go +++ b/pkg/detectors/weatherbit/weatherbit.go @@ -3,10 +3,11 @@ package weatherbit import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/weatherstack/weatherstack.go b/pkg/detectors/weatherstack/weatherstack.go index 685d360d710d..e0778f522ddd 100644 --- a/pkg/detectors/weatherstack/weatherstack.go +++ b/pkg/detectors/weatherstack/weatherstack.go @@ -3,11 +3,12 @@ package weatherstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -47,24 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.weatherstack.com/current?access_key=%s&query=LA", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - validResponse := strings.Contains(body, `location`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - if (res.StatusCode >= 200 && res.StatusCode < 300) && validResponse { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -73,6 +59,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.weatherstack.com/current?access_key=%s&query=LA", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + validResponse := strings.Contains(body, `location`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) + if (res.StatusCode >= 200 && res.StatusCode < 300) && validResponse { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_WeatherStack } diff --git a/pkg/detectors/web3storage/web3storage.go b/pkg/detectors/web3storage/web3storage.go index 06516a6629ce..988fabecfa9d 100644 --- a/pkg/detectors/web3storage/web3storage.go +++ b/pkg/detectors/web3storage/web3storage.go @@ -3,10 +3,11 @@ package web3storage import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 { diff --git a/pkg/detectors/webex/webex.go b/pkg/detectors/webex/webex.go index 67bacc452cbe..e3e9ec382c63 100644 --- a/pkg/detectors/webex/webex.go +++ b/pkg/detectors/webex/webex.go @@ -3,11 +3,12 @@ package webex import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() if err == nil { var message struct { Message string `json:"message"` diff --git a/pkg/detectors/webflow/webflow.go b/pkg/detectors/webflow/webflow.go index 3a65cb603a94..3dd5236893ee 100644 --- a/pkg/detectors/webflow/webflow.go +++ b/pkg/detectors/webflow/webflow.go @@ -3,10 +3,11 @@ package webflow import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -54,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/webscraper/webscraper.go b/pkg/detectors/webscraper/webscraper.go index 13319c8fdd89..b03edf540c1e 100644 --- a/pkg/detectors/webscraper/webscraper.go +++ b/pkg/detectors/webscraper/webscraper.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/webscraping/webscraping.go b/pkg/detectors/webscraping/webscraping.go index ebf57bee609c..d39d19169224 100644 --- a/pkg/detectors/webscraping/webscraping.go +++ b/pkg/detectors/webscraping/webscraping.go @@ -2,11 +2,12 @@ package webscraping import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -46,23 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", `https://api.webscrapingapi.com/v1?api_key=`+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - if strings.Contains(body, "key `url` required.") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -71,6 +58,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", `https://api.webscrapingapi.com/v1?api_key=`+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + if strings.Contains(body, "key `url` required.") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Webscraping } diff --git a/pkg/detectors/websitepulse/websitepulse.go b/pkg/detectors/websitepulse/websitepulse.go index 2a84454eddf0..2c890d27f34c 100644 --- a/pkg/detectors/websitepulse/websitepulse.go +++ b/pkg/detectors/websitepulse/websitepulse.go @@ -3,11 +3,12 @@ package websitepulse import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,23 +53,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result SecretParts: map[string]string{"key": resMatch}, } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.websitepulse.com/textserver.php?method=GetContacts&username=%s&key=%s", resIdMatch, resMatch), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - - if strings.Contains(body, "Active") { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resIdMatch, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resIdMatch, resMatch) } results = append(results, s1) @@ -79,6 +66,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resIdMatch string, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.websitepulse.com/textserver.php?method=GetContacts&username=%s&key=%s", resIdMatch, resMatch), nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + + if strings.Contains(body, "Active") { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Websitepulse } diff --git a/pkg/detectors/weightsandbiases/weightsandbiases.go b/pkg/detectors/weightsandbiases/base.go similarity index 63% rename from pkg/detectors/weightsandbiases/weightsandbiases.go rename to pkg/detectors/weightsandbiases/base.go index a175ed948ec7..4f5f76c7da51 100644 --- a/pkg/detectors/weightsandbiases/weightsandbiases.go +++ b/pkg/detectors/weightsandbiases/base.go @@ -17,53 +17,46 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -type Scanner struct{ client *http.Client } - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"wandb"}) + `\b([0-9a-f]{40})\b`) -) +// BaseScanner is a base struct embedded by versioned scanners. It holds the HTTP client and +// shared detection/verification logic. +type BaseScanner struct { + Client *http.Client +} -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { return []string{"wandb"} } +var defaultClient = common.SaneHttpClient() -// FromData will find and optionally verify Weightsandbiases secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { +// FromData finds and optionally verifies WeightsAndBiases secrets in data using the provided +// pattern. version is included in ExtraData of each result. +func (s BaseScanner) FromData(ctx context.Context, verify bool, data []byte, keyPat *regexp.Regexp, version int) ([]detectors.Result, error) { dataStr := string(data) - uniqueMatches := make(map[string]struct{}) for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { uniqueMatches[match[1]] = struct{}{} } + var results []detectors.Result for match := range uniqueMatches { - s1 := detectors.Result{ + r := detectors.Result{ DetectorType: detector_typepb.DetectorType_WeightsAndBiases, Raw: []byte(match), SecretParts: map[string]string{"key": match}, } if verify { - client := s.client - if client == nil { - client = defaultClient - } - - isVerified, extraData, verificationErr := verifyMatch(ctx, client, match) - s1.Verified = isVerified - s1.ExtraData = extraData - s1.SetVerificationError(verificationErr, match) + isVerified, extraData, verificationErr := s.verifyMatch(ctx, match) + r.Verified = isVerified + r.ExtraData = extraData + r.SetVerificationError(verificationErr, match) } - results = append(results, s1) - } + if r.ExtraData == nil { + r.ExtraData = make(map[string]string) + } + r.ExtraData["version"] = strconv.Itoa(version) - return + results = append(results, r) + } + return results, nil } type viewerResponse struct { @@ -77,7 +70,16 @@ type viewerResponse struct { } `json:"data"` } -func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) { +// verifyMatch checks the credential against the W&B GraphQL /graphql endpoint using the viewer query, +// which requires no special permissions. A 200 with a non-empty username means the token is valid; +// 401 means invalid or revoked. +// Docs: https://docs.wandb.ai/ref/graphql +func (s BaseScanner) verifyMatch(ctx context.Context, token string) (bool, map[string]string, error) { + client := s.Client + if client == nil { + client = defaultClient + } + query := `{"query": "query Viewer { viewer { id username email admin } }"}` const baseURL = "https://api.wandb.ai/graphql" @@ -124,10 +126,10 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, } } -func (s Scanner) Description() string { +func (s BaseScanner) Description() string { return "Weights & Biases is a Machine Learning Operations (MLOps) platform that helps track experiments, version datasets, evaluate model performance, and collaborate with team members" } -func (s Scanner) Type() detector_typepb.DetectorType { +func (s BaseScanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_WeightsAndBiases } diff --git a/pkg/detectors/weightsandbiases/v1/weightsandbiases.go b/pkg/detectors/weightsandbiases/v1/weightsandbiases.go new file mode 100644 index 000000000000..edb3498f898a --- /dev/null +++ b/pkg/detectors/weightsandbiases/v1/weightsandbiases.go @@ -0,0 +1,29 @@ +package weightsandbiases + +import ( + "context" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + base "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases" +) + +type Scanner struct { + base.BaseScanner +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +var keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"wandb"}) + `\b([0-9a-f]{40})\b`) + +func (s Scanner) Version() int { return 1 } + +func (s Scanner) Keywords() []string { return []string{"wandb"} } + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { + return s.BaseScanner.FromData(ctx, verify, data, keyPat, s.Version()) +} + diff --git a/pkg/detectors/weightsandbiases/weightsandbiases_integration_test.go b/pkg/detectors/weightsandbiases/v1/weightsandbiases_integration_test.go similarity index 87% rename from pkg/detectors/weightsandbiases/weightsandbiases_integration_test.go rename to pkg/detectors/weightsandbiases/v1/weightsandbiases_integration_test.go index 0b7a204dde5f..618b7267cdeb 100644 --- a/pkg/detectors/weightsandbiases/weightsandbiases_integration_test.go +++ b/pkg/detectors/weightsandbiases/v1/weightsandbiases_integration_test.go @@ -14,6 +14,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + base "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -56,6 +57,7 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { "admin": "false", "email": "source-integrations@trufflesec.com", "username": "source-integrations", + "version": "1", }, }, }, @@ -74,6 +76,7 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_WeightsAndBiases, Verified: false, + ExtraData: map[string]string{"version": "1"}, }, }, wantErr: false, @@ -93,7 +96,7 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { }, { name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + s: Scanner{BaseScanner: base.BaseScanner{Client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a weightsandbiases secret wandb %s within", secret)), @@ -103,6 +106,7 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_WeightsAndBiases, Verified: false, + ExtraData: map[string]string{"version": "1"}, }, }, wantErr: false, @@ -110,7 +114,7 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { }, { name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + s: Scanner{BaseScanner: base.BaseScanner{Client: common.ConstantResponseHttpClient(404, "")}}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a weightsandbiases secret wandb %s within", secret)), @@ -120,6 +124,7 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_WeightsAndBiases, Verified: false, + ExtraData: map[string]string{"version": "1"}, }, }, wantErr: false, @@ -138,11 +143,14 @@ func TestWeightsandbiases_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: \n %+v", got[i]) + } if (got[i].VerificationError() != nil) != tt.wantVerificationErr { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Weightsandbiases.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/weightsandbiases/weightsandbiases_test.go b/pkg/detectors/weightsandbiases/v1/weightsandbiases_test.go similarity index 100% rename from pkg/detectors/weightsandbiases/weightsandbiases_test.go rename to pkg/detectors/weightsandbiases/v1/weightsandbiases_test.go diff --git a/pkg/detectors/weightsandbiases/v2/weightsandbiases.go b/pkg/detectors/weightsandbiases/v2/weightsandbiases.go new file mode 100644 index 000000000000..fbf0cf210376 --- /dev/null +++ b/pkg/detectors/weightsandbiases/v2/weightsandbiases.go @@ -0,0 +1,31 @@ +package weightsandbiases + +import ( + "context" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + base "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases" +) + +type Scanner struct { + base.BaseScanner +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.Versioner = (*Scanner)(nil) + +// wandb_v1_<27 alphanumeric chars>_<49 alphanumeric chars> +// Example: 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 +var keyPat = regexp.MustCompile(`\b(wandb_v1_[A-Za-z0-9]{27}_[A-Za-z0-9]{49})\b`) + +func (s Scanner) Version() int { return 2 } + +func (s Scanner) Keywords() []string { return []string{"wandb_v1_"} } + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { + return s.BaseScanner.FromData(ctx, verify, data, keyPat, s.Version()) +} + diff --git a/pkg/detectors/weightsandbiases/v2/weightsandbiases_integration_test.go b/pkg/detectors/weightsandbiases/v2/weightsandbiases_integration_test.go new file mode 100644 index 000000000000..2499cf64bf50 --- /dev/null +++ b/pkg/detectors/weightsandbiases/v2/weightsandbiases_integration_test.go @@ -0,0 +1,175 @@ +//go:build detectors +// +build detectors + +package weightsandbiases + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + base "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestWeightsandbiasesV2_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("WEIGHTSANDBIASES_V2") + inactiveSecret := "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" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a weightsandbiases secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_WeightsAndBiases, + Verified: true, + ExtraData: map[string]string{ + "admin": "false", + "email": "muneeb.khan@trufflesec.com", + "username": "muneeb-khan-222", + "version": "2", + }, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a weightsandbiases secret %s within but not valid", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_WeightsAndBiases, + Verified: false, + ExtraData: map[string]string{"version": "2"}, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{BaseScanner: base.BaseScanner{Client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a weightsandbiases secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_WeightsAndBiases, + Verified: false, + ExtraData: map[string]string{"version": "2"}, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{BaseScanner: base.BaseScanner{Client: common.ConstantResponseHttpClient(404, "")}}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a weightsandbiases secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_WeightsAndBiases, + Verified: false, + ExtraData: map[string]string{"version": "2"}, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("WeightsandbiasesV2.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret", "SecretParts", "chunkOffset", "chunkOffsetSet") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("WeightsandbiasesV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/weightsandbiases/v2/weightsandbiases_test.go b/pkg/detectors/weightsandbiases/v2/weightsandbiases_test.go new file mode 100644 index 000000000000..32ce249eea87 --- /dev/null +++ b/pkg/detectors/weightsandbiases/v2/weightsandbiases_test.go @@ -0,0 +1,88 @@ +package weightsandbiases + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestWeightsandbiasesV2_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "typical pattern", + input: "WANDB_API_KEY = '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'", + want: []string{"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"}, + }, + { + name: "finds all matches", + input: `WANDB_API_KEY = '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' +WANDB_API_KEY = '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'`, + want: []string{ + "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", + "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", + }, + }, + { + name: "invalid pattern - wrong length", + input: "WANDB_API_KEY = 'wandb_v1_CNskTdKUs0f1uHZ4eOECFLof6aC_4IlqrKmMuTTfwXd5n6hf8VvcOX67MNii'", + want: []string{}, + }, + { + name: "does not match v1 legacy format", + input: "WANDB_API_KEY = 'eedf1c984f6b995ec40ecc6658356044847ffb31'", + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(test.want) > 0 && len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/wepay/wepay.go b/pkg/detectors/wepay/wepay.go index 6c231601416a..341f960c29c4 100644 --- a/pkg/detectors/wepay/wepay.go +++ b/pkg/detectors/wepay/wepay.go @@ -2,10 +2,11 @@ package wepay import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -66,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/whoxy/whoxy.go b/pkg/detectors/whoxy/whoxy.go index f84e7efe9e95..9ce58d9fc66b 100644 --- a/pkg/detectors/whoxy/whoxy.go +++ b/pkg/detectors/whoxy/whoxy.go @@ -3,11 +3,12 @@ package whoxy import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -47,23 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.whoxy.com/?key=%s&account=balance", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - continue - } - body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"status": 1`) { - s1.Verified = true - } - } + isVerified, verificationErr := verifyMatch(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } results = append(results, s1) @@ -72,6 +59,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyMatch(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.whoxy.com/?key=%s&account=balance", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + body := string(bodyBytes) + if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"status": 1`) { + return true, nil + } + + return false, nil +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Whoxy } diff --git a/pkg/detectors/wistia/wistia.go b/pkg/detectors/wistia/wistia.go index 846565df1b90..80c7cda03ca7 100644 --- a/pkg/detectors/wistia/wistia.go +++ b/pkg/detectors/wistia/wistia.go @@ -2,10 +2,11 @@ package wistia import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/wit/wit.go b/pkg/detectors/wit/wit.go index fb1829a0d432..b40a1627d6f8 100644 --- a/pkg/detectors/wit/wit.go +++ b/pkg/detectors/wit/wit.go @@ -3,10 +3,12 @@ package wit import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" + "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -31,39 +33,75 @@ func (s Scanner) Keywords() []string { } // FromData will find and optionally verify Wit secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + dataStr := string(data) - for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) + uniqueKeys := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueKeys[strings.TrimSpace(match[1])] = struct{}{} + } - s1 := detectors.Result{ + for key := range uniqueKeys { + result := detectors.Result{ DetectorType: detector_typepb.DetectorType_Wit, - Raw: []byte(resMatch), - SecretParts: map[string]string{"key": resMatch}, + Raw: []byte(key), + SecretParts: map[string]string{ + "key": key, + }, } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.wit.ai/message?q=saascndncdcdksCHDKSCVSDCasdasdVCSDVCSDAVHKCDCVHKSADVCKDVKCDSVHCSACVHJDSCVJHSADCVJHSAJ", nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + verified, verificationErr := verifyWitKey(ctx, client, key) + result.SetVerificationError(verificationErr, key) + result.Verified = verified } - results = append(results, s1) + results = append(results, result) } - return results, nil + return +} + +func verifyWitKey( + ctx context.Context, + client *http.Client, + key string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://api.wit.ai/apps?offset=1&limit=2", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Bearer "+key) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/wit/wit_integration_test.go b/pkg/detectors/wit/wit_integration_test.go index 12a40027ca1d..bf45af401c40 100644 --- a/pkg/detectors/wit/wit_integration_test.go +++ b/pkg/detectors/wit/wit_integration_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -96,7 +96,17 @@ func TestWit_FromChunk(t *testing.T) { } got[i].Raw = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Wit.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/worksnaps/worksnaps.go b/pkg/detectors/worksnaps/worksnaps.go index 8753398356e5..2ebef27265af 100644 --- a/pkg/detectors/worksnaps/worksnaps.go +++ b/pkg/detectors/worksnaps/worksnaps.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -56,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/workstack/workstack.go b/pkg/detectors/workstack/workstack.go index 37883a01a940..0a3d38b32d4e 100644 --- a/pkg/detectors/workstack/workstack.go +++ b/pkg/detectors/workstack/workstack.go @@ -2,10 +2,11 @@ package workstack import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Accept", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/worldcoinindex/worldcoinindex.go b/pkg/detectors/worldcoinindex/worldcoinindex.go index 92783c8e719a..1a74179a0c2e 100644 --- a/pkg/detectors/worldcoinindex/worldcoinindex.go +++ b/pkg/detectors/worldcoinindex/worldcoinindex.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"Markets"`) - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { if validResponse { s1.Verified = true diff --git a/pkg/detectors/worldweather/worldweather.go b/pkg/detectors/worldweather/worldweather.go index 596c2d2d03a6..bba10a2cb693 100644 --- a/pkg/detectors/worldweather/worldweather.go +++ b/pkg/detectors/worldweather/worldweather.go @@ -3,10 +3,11 @@ package worldweather import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/wrike/wrike.go b/pkg/detectors/wrike/wrike.go index a0017ab1c0c8..146b00c8e158 100644 --- a/pkg/detectors/wrike/wrike.go +++ b/pkg/detectors/wrike/wrike.go @@ -3,10 +3,11 @@ package wrike import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/yandex/yandex.go b/pkg/detectors/yandex/yandex.go index a021e52042c0..91e06b68c4db 100644 --- a/pkg/detectors/yandex/yandex.go +++ b/pkg/detectors/yandex/yandex.go @@ -2,10 +2,11 @@ package yandex import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/yelp/yelp.go b/pkg/detectors/yelp/yelp.go index a5577b6670de..b287afd8eb53 100644 --- a/pkg/detectors/yelp/yelp.go +++ b/pkg/detectors/yelp/yelp.go @@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else if res.StatusCode == 401 || res.StatusCode == 403 { diff --git a/pkg/detectors/youneedabudget/youneedabudget.go b/pkg/detectors/youneedabudget/youneedabudget.go index 49a83193e610..3cb64d8e000a 100644 --- a/pkg/detectors/youneedabudget/youneedabudget.go +++ b/pkg/detectors/youneedabudget/youneedabudget.go @@ -3,10 +3,11 @@ package youneedabudget import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/yousign/yousign.go b/pkg/detectors/yousign/yousign.go index 973fd4b59029..546fa4250b3b 100644 --- a/pkg/detectors/yousign/yousign.go +++ b/pkg/detectors/yousign/yousign.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } else { @@ -71,7 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) res, err = client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/youtubeapikey/youtubeapikey.go b/pkg/detectors/youtubeapikey/youtubeapikey.go index d0f6dffa12e7..3f28f63601ec 100644 --- a/pkg/detectors/youtubeapikey/youtubeapikey.go +++ b/pkg/detectors/youtubeapikey/youtubeapikey.go @@ -2,10 +2,11 @@ package youtubeapikey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -58,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/engine/ahocorasick/README.md b/pkg/engine/ahocorasick/README.md new file mode 100644 index 000000000000..9ee46216804a --- /dev/null +++ b/pkg/engine/ahocorasick/README.md @@ -0,0 +1,100 @@ +# ahocorasick + +This package decides **which detectors are worth running on a piece of data**. + +## Why it exists + +TruffleHog ships hundreds of detectors. Running every one against every chunk of +scanned data would be far too slow, and almost always pointless: a file about +billing has nothing to do with a Kubernetes token. + +So each detector declares a few keywords it cares about. A SendGrid key always +contains `SG.`, so the SendGrid detector only needs to see chunks containing +`SG.`. This package checks the whole keyword list in one pass and reports which +detectors matched. Everything else is skipped. + +That check is a **prefilter**: a cheap test run before the expensive one. + +## How it works + +The [Aho-Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) +searches for many words at once. Rather than scanning the text once per keyword, +it builds a lookup structure from every keyword up front, then walks the text a +single time. The cost barely changes whether there are ten keywords or ten +thousand. + +`FindDetectorMatches` does the work: + +1. Make a lowercased copy of the chunk, so matching ignores case. +2. Walk the copy once and collect every keyword hit. +3. For each hit, work out a **span**: a window of bytes around it, cut from the + original chunk. Detectors get this window rather than the whole chunk, which + keeps their regexes fast. Some detectors ask for a wider window, for example + when a credential comes in two parts that may sit on separate lines. +4. Return one entry per matching detector, with its spans merged. + +## Lowercasing only A-Z + +Step 1 lowercases `A-Z` by hand and leaves every other byte untouched, rather +than using `bytes.ToLower`. + +The reason is step 3. Positions come from the lowercased copy, but the bytes are +cut from the original chunk. That only lines up while both are the same length. + +Full Unicode lowercasing does not promise that. Some characters need a different +number of bytes once lowercased — `İ` takes two bytes, `i` takes one. A single +one of those makes the copy shorter, so every position after it points slightly +too far back, and a detector is handed text that is not the text that matched. + +Changing only `A-Z` always keeps the same length, so positions stay correct. + +Keywords are folded through the same function when they are registered, so both +sides of the search always agree. + +### Known limitation + +A cased non-ASCII letter is left exactly as written, on both sides. So a keyword +containing one matches only text spelling that letter the same way: a keyword +`CAFÉ` matches `CAFÉ` but not `café`. + +Every built-in detector keyword is plain ASCII, so this cannot affect them, and +a test checks that assumption still holds. It only applies to a custom detector +using a non-ASCII keyword. + +## The buffer pool + +Every chunk needs its own lowercased copy, and each copy is discarded moments +later. Allocating a fresh one each time would mean throwing away a chunk-sized +buffer on every call, for every chunk in a scan. + +Instead the copies come from `lowerBufPool`, a small pool of reusable buffers: + +- `getLowerBuf` borrows one, growing it only if the chunk does not fit. +- `putLowerBuf` hands it back once the chunk is done with. + +In a steady scan the pool already holds a buffer large enough, so making the copy +costs **no allocation at all**. Buffers larger than `maxPooledLowerBuf` are +dropped rather than kept, so one unusually big chunk cannot leave a large buffer +parked in the pool for the rest of the run. + +## What it costs + +`BenchmarkLowerBuf` compares borrowing a buffer from the pool against building a +fresh lowercased copy each time. On an Apple M3 Pro: + +``` +BenchmarkLowerBuf/pooled-11 245199 4929 ns/op 1866.34 MB/s 0 B/op 0 allocs/op +BenchmarkLowerBuf/unpooled-11 225938 5097 ns/op 1805.07 MB/s 9472 B/op 1 allocs/op +``` + +The time difference is small enough to be run-to-run noise. What the pool +actually removes is the allocation: **9472 B and 1 allocation per chunk, down to +zero**. That copy would otherwise be made and thrown away for every chunk in a +scan, so the saving grows with the amount of data scanned rather than showing up +on the clock here. + +Reproduce with: + +```sh +go test ./pkg/engine/ahocorasick/ -bench=BenchmarkLowerBuf -benchmem +``` diff --git a/pkg/engine/ahocorasick/ahocorasickcore.go b/pkg/engine/ahocorasick/ahocorasickcore.go index 11a63b74e5a9..93210958968e 100644 --- a/pkg/engine/ahocorasick/ahocorasickcore.go +++ b/pkg/engine/ahocorasick/ahocorasickcore.go @@ -1,8 +1,7 @@ package ahocorasick import ( - "bytes" - "strings" + "sync" ahocorasick "github.com/BobuSumisu/aho-corasick" @@ -11,6 +10,74 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) +// lowerBuf is a reusable buffer holding the lowercased copy of one chunk. +// +// Every chunk needs such a copy, and each one is thrown away moments later, so +// rather than allocate a new buffer every time we borrow one from a pool and +// hand it back afterwards. In a steady scan the pool already holds a buffer big +// enough, so the copy costs no allocation at all. +// +// The pool stores pointers rather than plain slices. Putting a slice into a +// sync.Pool would allocate on every Put, which is the very thing being avoided. +type lowerBuf struct{ b []byte } + +var lowerBufPool = sync.Pool{New: func() any { return new(lowerBuf) }} + +// maxPooledLowerBuf is the largest buffer worth keeping for reuse. +// +// One unusually large chunk would otherwise leave a matching buffer sitting in +// the pool for the rest of the run, once per worker goroutine. Past this size +// it is cheaper to let the buffer go and allocate again next time. +const maxPooledLowerBuf = 1 << 20 // 1 MiB + +// getLowerBuf borrows a buffer able to hold size bytes. +func getLowerBuf(size int) *lowerBuf { + buf := lowerBufPool.Get().(*lowerBuf) + if cap(buf.b) < size { + buf.b = make([]byte, 0, size) + } + return buf +} + +// putLowerBuf returns a buffer for reuse, unless it is too big to be worth +// holding on to. +func putLowerBuf(buf *lowerBuf) { + if cap(buf.b) > maxPooledLowerBuf { + return + } + buf.b = buf.b[:0] + lowerBufPool.Put(buf) +} + +// appendASCIILower copies src onto dst, turning A-Z into a-z and leaving every +// other byte exactly as it was. +// +// Only A-Z is changed, for two reasons. +// +// It keeps the copy the same length as the original. Lowercasing a character +// can change how many bytes it takes: "İ" needs two bytes, "i" needs one. +// FindDetectorMatches searches the lowercased copy but then cuts bytes out of +// the original chunk, so if the copy were a different length every position +// would point somewhere slightly wrong, and the text handed to a detector would +// not be the text that matched. +// +// Nothing is lost by leaving other bytes alone. Every built-in detector keyword +// is plain ASCII, so a non-ASCII byte can never be part of a match either way. +// +// Keywords are folded through this same function when they are registered, so +// both sides of the search always agree. The one gap is a custom detector whose +// keyword contains a cased non-ASCII letter: that letter is left as written on +// both sides, so the keyword matches only text spelling it the same way. +func appendASCIILower(dst, src []byte) []byte { + for _, c := range src { + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + dst = append(dst, c) + } + return dst +} + // DetectorKey is used to identify a detector in the keywordsToDetectors map. // Multiple detectors can have the same detector type but different versions. // This allows us to identify a detector by its type and version. An @@ -146,9 +213,11 @@ func NewAhoCorasickCore(allDetectors []detectors.Detector, opts ...CoreOption) * key := CreateDetectorKey(d) detectorsByKey[key] = d for _, kw := range d.Keywords() { - kwLower := strings.ToLower(kw) - keywords = append(keywords, kwLower) - keywordsToDetectors[kwLower] = append(keywordsToDetectors[kwLower], key) + // Fold the keyword exactly the way FindDetectorMatches folds the + // chunk it searches, so the two sides can never disagree. + kwFolded := string(appendASCIILower(nil, []byte(kw))) + keywords = append(keywords, kwFolded) + keywordsToDetectors[kwFolded] = append(keywordsToDetectors[kwFolded], key) } } @@ -239,7 +308,14 @@ func (d *DetectorMatch) Matches() [][]byte { return d.matches } // // The matches field contains the actual byte slices of the matched portions from the chunk data. func (ac *Core) FindDetectorMatches(chunkData []byte) []*DetectorMatch { - matches := ac.prefilter.Match(bytes.ToLower(chunkData)) + // Search a lowercased copy so matching ignores case, but keep reporting + // positions against the original chunk, which is what the spans below cut + // from. The copy is borrowed and returned so it costs no allocation. + lowered := getLowerBuf(len(chunkData)) + defer putLowerBuf(lowered) + lowered.b = appendASCIILower(lowered.b[:0], chunkData) + + matches := ac.prefilter.Match(lowered.b) matchCount := len(matches) if matchCount == 0 { diff --git a/pkg/engine/ahocorasick/ahocorasickcore_test.go b/pkg/engine/ahocorasick/ahocorasickcore_test.go index 5828a846bf67..18080a598e31 100644 --- a/pkg/engine/ahocorasick/ahocorasickcore_test.go +++ b/pkg/engine/ahocorasick/ahocorasickcore_test.go @@ -1,13 +1,19 @@ package ahocorasick import ( + "bytes" "context" + "strings" + "sync" "testing" + "unicode/utf8" "github.com/stretchr/testify/assert" + "pgregory.net/rapid" "github.com/trufflesecurity/trufflehog/v3/pkg/custom_detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -434,3 +440,265 @@ func TestFindDetectorMatches(t *testing.T) { }) } } + +// --------------------------------------------------------------------------- +// Case folding +// +// FindDetectorMatches searches a lowercased copy of the chunk, then cuts spans +// out of the original. That only works while the copy is the same length as +// the original, which is why only A-Z is lowered. +// --------------------------------------------------------------------------- + +// keywordDetector matches on one caller-chosen keyword. +type keywordDetector struct{ keyword string } + +func (keywordDetector) FromData(context.Context, bool, []byte) ([]detectors.Result, error) { + return nil, nil +} +func (k keywordDetector) Keywords() []string { return []string{k.keyword} } +func (keywordDetector) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_Generic +} +func (keywordDetector) Description() string { return "" } + +func TestAppendASCIILower(t *testing.T) { + // Must agree with the standard library across the whole ASCII range. + var ascii []byte + for c := range 128 { + ascii = append(ascii, byte(c)) + } + assert.Equal(t, bytes.ToLower(ascii), appendASCIILower(nil, ascii)) + + // Must never change length, unlike bytes.ToLower. Each of these shrinks + // when lowercased the normal way, which is what would move the spans. + for _, s := range []string{"İ", "ẞ", "İİİ", "prefix ẞ suffix"} { + assert.Len(t, appendASCIILower(nil, []byte(s)), len(s), "input %q", s) + assert.NotEqual(t, len(s), len(bytes.ToLower([]byte(s))), + "bytes.ToLower(%q) no longer changes length, so this case proves nothing", s) + } +} + +func TestFindDetectorMatchesIsCaseInsensitive(t *testing.T) { + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{"sendgrid"}}) + for _, chunk := range []string{"sendgrid", "SENDGRID", "SendGrid"} { + assert.Len(t, core.FindDetectorMatches([]byte(chunk)), 1, "chunk %q", chunk) + } +} + +// Multi-byte characters before a keyword must not move the span off it. +func TestFindDetectorMatchesSpansSurviveMultibyte(t *testing.T) { + const keyword = "sekrit" + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{keyword}}) + + // The run of "İ" is longer than the window the span calculator adds around + // a hit, otherwise the window would cover the keyword regardless and a + // wrong position would go unnoticed. + chunk := []byte(strings.Repeat("İ", 700) + " padding " + keyword + " trailing") + + matches := core.FindDetectorMatches(chunk) + assert.Len(t, matches, 1) + for _, m := range matches[0].matches { + assert.Contains(t, string(m), keyword) + } +} + +// Whatever surrounds a keyword, the span cut for it must stay inside the chunk +// and must still cover the keyword that produced it. +// +// Positions come from the lowercased copy while the bytes are cut from the +// original, so this holds only while folding preserves length. The generator +// leans on characters where that is easy to get wrong: "İ" and "ẞ" both shrink +// under full Unicode lowercasing, and the raw tail is arbitrary bytes that need +// not be valid UTF-8. +func TestFindDetectorMatchesSpanProperties(t *testing.T) { + const keyword = "token" + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{keyword}}) + + // Padding is built from long runs of one character rather than a random + // mix, so it reliably grows past the window the span calculator adds around + // a hit. A drifting position is only observable outside that window. + interesting := []rune{'a', 'Z', ' ', 'é', 'İ', 'ẞ', 'Κ', '密'} + padding := rapid.Custom(func(t *rapid.T) string { + var sb strings.Builder + for range rapid.IntRange(0, 3).Draw(t, "runs") { + sb.WriteString(strings.Repeat( + string(rapid.SampledFrom(interesting).Draw(t, "char")), + rapid.IntRange(0, 600).Draw(t, "runLength"), + )) + } + return sb.String() + }) + + rapid.Check(t, func(t *rapid.T) { + chunk := []byte(padding.Draw(t, "prefix") + keyword + padding.Draw(t, "suffix")) + chunk = append(chunk, rapid.SliceOf(rapid.Byte()).Draw(t, "tail")...) + + matches := core.FindDetectorMatches(chunk) + if len(matches) != 1 { + t.Fatalf("keyword present but got %d detector matches", len(matches)) + } + + spans := matches[0].matchSpans + if len(spans) == 0 { + t.Fatal("detector matched but no spans were produced") + } + for _, span := range spans { + if span.startOffset < 0 || span.startOffset > span.endOffset || + span.endOffset > int64(len(chunk)) { + t.Fatalf("span [%d,%d) outside chunk of %d bytes", + span.startOffset, span.endOffset, len(chunk)) + } + // Every span comes from at least one hit, so slicing the original + // chunk with it must give back text containing the keyword. + text := string(chunk[span.startOffset:span.endOffset]) + if !strings.Contains(strings.ToLower(text), keyword) { + t.Fatalf("span [%d,%d) does not cover the keyword", + span.startOffset, span.endOffset) + } + } + }) +} + +// A buffer coming back from the pool still holds the previous chunk's bytes, +// so a later, shorter chunk must not be able to see them. +func TestLowerBufPoolReuseIsClean(t *testing.T) { + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{"token"}}) + + long := []byte(strings.Repeat("x", 4096) + "token" + strings.Repeat("y", 4096)) + assert.Len(t, core.FindDetectorMatches(long), 1) + + short := []byte("token") + matches := core.FindDetectorMatches(short) + assert.Len(t, matches, 1) + for _, m := range matches[0].matchSpans { + assert.LessOrEqual(t, m.endOffset, int64(len(short)), + "leftover bytes from the previous chunk leaked through the pool") + } +} + +// The pool is shared by every scanner worker. +func TestFindDetectorMatchesConcurrent(t *testing.T) { + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{"token"}}) + chunks := [][]byte{ + []byte("token here"), + []byte(strings.Repeat("padding ", 500) + "token"), + []byte("İİİ token"), + } + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + for range 200 { + for _, c := range chunks { + for _, m := range core.FindDetectorMatches(c) { + for _, span := range m.matchSpans { + if span.endOffset > int64(len(c)) { + panic("span escaped the chunk") + } + } + } + } + } + }) + } + wg.Wait() +} + +// Lowering only A-Z is safe only while every keyword is ASCII, since a +// non-ASCII byte then cannot be part of any match. +func TestAllDefaultKeywordsAreASCII(t *testing.T) { + for _, d := range defaults.DefaultDetectors() { + for _, kw := range d.Keywords() { + for i := range len(kw) { + assert.Less(t, kw[i], byte(utf8.RuneSelf), + "%v keyword %q is not ASCII", d.Type(), kw) + } + } + } +} + +// FuzzAppendASCIILower checks the two properties the spans rely on: the output +// is always the same length as the input, and only A-Z changes. +func FuzzAppendASCIILower(f *testing.F) { + for _, seed := range []string{"", "token", "TOKEN", "İ", "ẞ", "\x00\xff", "MiXeD 123"} { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, src []byte) { + got := appendASCIILower(nil, src) + if len(got) != len(src) { + t.Fatalf("length changed: %d -> %d", len(src), len(got)) + } + for i, c := range src { + want := c + if c >= 'A' && c <= 'Z' { + want = c + ('a' - 'A') + } + if got[i] != want { + t.Fatalf("byte %d: %q became %q, want %q", i, c, got[i], want) + } + } + }) +} + +// Keywords are folded the same way chunks are, so matching ignores ASCII case. +// A cased non-ASCII letter is left as written on both sides, so it matches only +// text spelling that letter the same way. Built-in keywords are all ASCII, so +// this only ever affects a custom detector. +func TestKeywordFoldingMatchesChunkFolding(t *testing.T) { + for _, tc := range []struct { + keyword string + matches []string + misses []string + }{ + // ASCII folds fully, so every spelling matches. + {"TOKEN", []string{"token", "TOKEN", "Token"}, nil}, + // The ASCII letters still fold; the "É" must be written as it was. + {"CAFÉ", []string{"CAFÉ", "cafÉ"}, []string{"café", "Café"}}, + // Scripts without letter case are unaffected. + {"密钥", []string{"密钥"}, nil}, + } { + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{tc.keyword}}) + for _, chunk := range tc.matches { + assert.Len(t, core.FindDetectorMatches([]byte(chunk)), 1, + "keyword %q should match chunk %q", tc.keyword, chunk) + } + for _, chunk := range tc.misses { + assert.Empty(t, core.FindDetectorMatches([]byte(chunk)), + "keyword %q should not match chunk %q", tc.keyword, chunk) + } + } +} + +// A keyword is registered under one spelling, so a single occurrence reports a +// single match rather than one per spelling. +func TestKeywordRegisteredOnce(t *testing.T) { + core := NewAhoCorasickCore([]detectors.Detector{keywordDetector{"token"}}) + assert.Len(t, core.keywordsToDetectors["token"], 1) + + matches := core.FindDetectorMatches([]byte("token")) + assert.Len(t, matches, 1) + assert.Len(t, matches[0].matchSpans, 1, "keyword counted twice") +} + +// BenchmarkLowerBuf shows what the pool is for: borrowing a buffer costs no +// allocation, while building a fresh lowercased copy allocates one per chunk. +func BenchmarkLowerBuf(b *testing.B) { + data := []byte(strings.Repeat("The Quick Brown Fox Jumps Over A Sleeping Dog\n", 200)) + + b.Run("pooled", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(data))) + for range b.N { + buf := getLowerBuf(len(data)) + buf.b = appendASCIILower(buf.b[:0], data) + putLowerBuf(buf) + } + }) + b.Run("unpooled", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(data))) + for range b.N { + _ = appendASCIILower(make([]byte, 0, len(data)), data) + } + }) +} diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index f769af2c5614..ee5dc6351dd9 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -1,6 +1,8 @@ package defaults import ( + "slices" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abyssale" @@ -44,7 +46,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appcues" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appfollow" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appointedd" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appoptics" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apptivo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/artifactory" @@ -56,6 +57,9 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atera" atlassianv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassian/v1" atlassianv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassian/v2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter/bitbucketdatacenter" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter/confluencedatacenter" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter/jiradatacenterpat" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/audd" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/auth0managementapitoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/auth0oauth" @@ -65,6 +69,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/avazapersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aviationstack" aws_access_keys "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws/access_keys" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws/appsync" aws_session_keys "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws/session_keys" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/axonaut" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aylien" @@ -93,7 +98,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/besttime" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/betterstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/billomat" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bingsubscriptionkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitbar" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitbucketapppassword" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitcoinaverage" @@ -109,6 +113,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/box" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/boxoauth" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/braintreepayments" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/braintrust" brandfetchv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/brandfetch/v1" brandfetchv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/brandfetch/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/browserstack" @@ -161,10 +166,13 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/closecrm" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudconvert" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudelements" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareapitoken" + cloudflareapitokenv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareapitoken/v1" + cloudflareapitokenv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareapitoken/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflarecakey" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareglobalapikey" + cloudflareglobalapikeyv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareglobalapikey/v1" + cloudflareglobalapikeyv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudflareglobalapikey/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudimage" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudinary" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudmersive" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudplan" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/cloudsmith" @@ -185,7 +193,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/commercejs" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/commodities" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/companyhub" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/confluencedatacenter" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/confluent" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/contentfulpersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/conversiontools" @@ -212,6 +219,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dareboost" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databox" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databrickstoken" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogapikey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datagov" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepai" @@ -245,6 +253,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dronahq" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/droneci" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dropbox" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/duffeltoken" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/duo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/duply" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dwolla" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dynalist" @@ -280,6 +290,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fibery" figmapersonalaccesstokenv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/figmapersonalaccesstoken/v1" figmapersonalaccesstokenv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/figmapersonalaccesstoken/v2" + figmapersonalaccesstokenv3 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/figmapersonalaccesstoken/v3" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fileio" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/finage" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/financialmodelingprep" @@ -333,6 +344,7 @@ import ( gitlabv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v1" gitlabv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v2" gitlabv3 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v3" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlaboauth2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitter" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glassnode" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocanvas" @@ -355,6 +367,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/happyscribe" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/harness" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/harvest" + hashicorpbatchtoken "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hashicorpvault/hashicorpvaultbatchtoken" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hashicorpvault/hashicorpvaulttoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hashicorpvaultauth" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hasura" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hellosign" @@ -373,6 +387,7 @@ import ( hubspot_apikey_v2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hubspot_apikey/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/humanity" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/humioapitoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hunter" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hybiscus" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/hypertrack" @@ -394,11 +409,11 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ip2location" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipgeolocation" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipinfo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipinfodb" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipquality" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jdbc" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiradatacenterpat" jiratokenv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiratoken/v1" jiratokenv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiratoken/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jotform" @@ -437,6 +452,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/liveagent" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/livestorm" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loadmill" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lob" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/locationiq" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loggly" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loginradius" @@ -472,7 +488,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/metaapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/metabase" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/metrilo" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/microsoftteamswebhook" + microsoftteamswebhookv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/microsoftteamswebhook/v1" + microsoftteamswebhookv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/microsoftteamswebhook/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/miro" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite" @@ -494,7 +511,13 @@ import ( netlifyv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netsuite" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/neutrinoapi" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicbrowserkey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsinsertkey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsquerykey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newreliclicensekey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicmobileapptoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicuserkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newsapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newscatcher" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nexmoapikey" @@ -515,6 +538,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nvapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nylas" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/oanda" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/octopusdeploy" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/okta" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/omnisend" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/onedesk" @@ -526,6 +550,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openai" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openaiadmin" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opencagedata" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openrouter" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openuv" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openvpn" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openweather" @@ -555,9 +580,11 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/peopledatalabs" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pepipost" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/percy" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pganalyzereadkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/photoroom" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/phraseaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pinata" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pinecone" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pipedream" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pipedrive" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/pivotaltracker" @@ -610,6 +637,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/reallysimplesystems" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rebrandly" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rechargepayments" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redhatpyxis" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redis" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/refiner" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rentman" @@ -617,8 +645,10 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/replicate" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/replyio" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/requestfinance" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/resend" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/restpackhtmltopdfapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/restpackscreenshotapi" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rev" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/revampcrm" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ringcentral" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ritekit" @@ -670,6 +700,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sheety" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sherpadesk" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shipday" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shippo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shodankey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shopify" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/shopifyoauth" @@ -699,9 +730,12 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snipcart" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snykkey" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sonarcloud" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/solarwindsobservability" + sonarcloudv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sonarcloud/v1" + sonarcloudv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sonarcloud/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spectralops" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/speechtextai" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/splunkobservabilitytoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular" @@ -764,6 +798,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timecamp" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timezoneapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tineswebhook" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tly" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tmetric" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/todoist" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tokeet" @@ -800,6 +835,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/upwave" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/uri" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/urlscan" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/user" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userflow" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vagrantcloudpersonaltoken" @@ -830,9 +866,11 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/webscraper" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/webscraping" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/websitepulse" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases" + weightsandbiasesv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases/v1" + weightsandbiasesv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/whoxy" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wistia" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wit" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wiz" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/worksnaps" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/workstack" @@ -860,11 +898,12 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zohocrm" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) func buildDetectorList() []detectors.Detector { - return []detectors.Detector{ + dets := []detectors.Detector{ &abyssale.Scanner{}, // &abstract.Scanner{}, &abuseipdb.Scanner{}, @@ -912,7 +951,7 @@ func buildDetectorList() []detectors.Detector { &appcues.Scanner{}, &appfollow.Scanner{}, &appointedd.Scanner{}, - &appoptics.Scanner{}, + &appsync.Scanner{}, &appsynergy.Scanner{}, &apptivo.Scanner{}, &artifactory.Scanner{}, @@ -964,8 +1003,8 @@ func buildDetectorList() []detectors.Detector { &besttime.Scanner{}, &betterstack.Scanner{}, &billomat.Scanner{}, - &bingsubscriptionkey.Scanner{}, &bitbar.Scanner{}, + &bitbucketdatacenter.Scanner{}, &bitbucketapppassword.Scanner{}, &bitcoinaverage.Scanner{}, &bitfinex.Scanner{}, @@ -981,6 +1020,7 @@ func buildDetectorList() []detectors.Detector { &box.Scanner{}, &boxoauth.Scanner{}, &braintreepayments.Scanner{}, + &braintrust.Scanner{}, &brandfetchv1.Scanner{}, &brandfetchv2.Scanner{}, &browserstack.Scanner{}, @@ -1033,10 +1073,13 @@ func buildDetectorList() []detectors.Detector { &closecrm.Scanner{}, &cloudconvert.Scanner{}, &cloudelements.Scanner{}, - &cloudflareapitoken.Scanner{}, + &cloudflareapitokenv1.Scanner{}, + &cloudflareapitokenv2.Scanner{}, &cloudflarecakey.Scanner{}, - &cloudflareglobalapikey.Scanner{}, + &cloudflareglobalapikeyv1.Scanner{}, + &cloudflareglobalapikeyv2.Scanner{}, &cloudimage.Scanner{}, + &cloudinary.Scanner{}, &cloudmersive.Scanner{}, &cloudplan.Scanner{}, &cloudsmith.Scanner{}, @@ -1085,6 +1128,7 @@ func buildDetectorList() []detectors.Detector { &dareboost.Scanner{}, &databox.Scanner{}, &databrickstoken.Scanner{}, + &datadogapikey.Scanner{}, &datadogtoken.Scanner{}, &datagov.Scanner{}, // &debounce.Scanner{}, @@ -1119,6 +1163,8 @@ func buildDetectorList() []detectors.Detector { &dronahq.Scanner{}, &droneci.Scanner{}, &dropbox.Scanner{}, + &duffeltoken.Scanner{}, + &duo.Scanner{}, &duply.Scanner{}, &dwolla.Scanner{}, &dynalist.Scanner{}, @@ -1154,6 +1200,7 @@ func buildDetectorList() []detectors.Detector { &fibery.Scanner{}, &figmapersonalaccesstokenv1.Scanner{}, &figmapersonalaccesstokenv2.Scanner{}, + &figmapersonalaccesstokenv3.Scanner{}, &fileio.Scanner{}, &finage.Scanner{}, &financialmodelingprep.Scanner{}, @@ -1207,6 +1254,7 @@ func buildDetectorList() []detectors.Detector { &githubapp.Scanner{}, &githubv1.Scanner{}, &githubv2.Scanner{}, + &gitlaboauth2.Scanner{}, &gitlabv1.Scanner{}, &gitlabv2.Scanner{}, &gitlabv3.Scanner{}, @@ -1232,7 +1280,9 @@ func buildDetectorList() []detectors.Detector { &happyscribe.Scanner{}, &harness.Scanner{}, &harvest.Scanner{}, + &hashicorpbatchtoken.Scanner{}, &hashicorpvaultauth.Scanner{}, + &hashicorpvaulttoken.Scanner{}, &hasura.Scanner{}, &hellosign.Scanner{}, &helpcrunch.Scanner{}, @@ -1251,6 +1301,7 @@ func buildDetectorList() []detectors.Detector { &hubspot_apikey_v2.Scanner{}, &huggingface.Scanner{}, &humanity.Scanner{}, + &humioapitoken.Scanner{}, &hunter.Scanner{}, &hybiscus.Scanner{}, &hypertrack.Scanner{}, @@ -1273,6 +1324,7 @@ func buildDetectorList() []detectors.Detector { &ip2location.Scanner{}, &ipapi.Scanner{}, &ipgeolocation.Scanner{}, + &ipinfo.Scanner{}, &ipinfodb.Scanner{}, &ipquality.Scanner{}, &ipstack.Scanner{}, @@ -1317,6 +1369,7 @@ func buildDetectorList() []detectors.Detector { &liveagent.Scanner{}, &livestorm.Scanner{}, &loadmill.Scanner{}, + &lob.Scanner{}, &locationiq.Scanner{}, &loggly.Scanner{}, &loginradius.Scanner{}, @@ -1355,7 +1408,8 @@ func buildDetectorList() []detectors.Detector { &metaapi.Scanner{}, &metabase.Scanner{}, &metrilo.Scanner{}, - µsoftteamswebhook.Scanner{}, + µsoftteamswebhookv1.Scanner{}, + µsoftteamswebhookv2.Scanner{}, &mindmeister.Scanner{}, &miro.Scanner{}, &mite.Scanner{}, @@ -1379,7 +1433,13 @@ func buildDetectorList() []detectors.Detector { &netlifyv2.Scanner{}, &netsuite.Scanner{}, &neutrinoapi.Scanner{}, + &newrelicbrowserkey.Scanner{}, + &newrelicinsightsinsertkey.Scanner{}, + &newrelicinsightsquerykey.Scanner{}, + &newreliclicensekey.Scanner{}, + &newrelicmobileapptoken.Scanner{}, &newrelicpersonalapikey.Scanner{}, + &newrelicuserkey.Scanner{}, &newsapi.Scanner{}, &newscatcher.Scanner{}, &nexmoapikey.Scanner{}, @@ -1400,6 +1460,7 @@ func buildDetectorList() []detectors.Detector { &nvapi.Scanner{}, &nylas.Scanner{}, &oanda.Scanner{}, + &octopusdeploy.Scanner{}, &okta.Scanner{}, &omnisend.Scanner{}, &onedesk.Scanner{}, @@ -1411,6 +1472,7 @@ func buildDetectorList() []detectors.Detector { &openai.Scanner{}, &openaiadmin.Scanner{}, &opencagedata.Scanner{}, + &openrouter.Scanner{}, &openuv.Scanner{}, &openvpn.Scanner{}, &openweather.Scanner{}, @@ -1440,9 +1502,11 @@ func buildDetectorList() []detectors.Detector { &peopledatalabs.Scanner{}, &pepipost.Scanner{}, &percy.Scanner{}, + &pganalyzereadkey.Scanner{}, &photoroom.Scanner{}, &phraseaccesstoken.Scanner{}, &pinata.Scanner{}, + &pinecone.Scanner{}, &pipedream.Scanner{}, &pipedrive.Scanner{}, &pivotaltracker.Scanner{}, @@ -1496,6 +1560,7 @@ func buildDetectorList() []detectors.Detector { &reallysimplesystems.Scanner{}, &rebrandly.Scanner{}, &rechargepayments.Scanner{}, + &redhatpyxis.Scanner{}, &redis.Scanner{}, &refiner.Scanner{}, &rentman.Scanner{}, @@ -1503,9 +1568,11 @@ func buildDetectorList() []detectors.Detector { &replicate.Scanner{}, &replyio.Scanner{}, &requestfinance.Scanner{}, + &resend.Scanner{}, // &restpack.Scanner{}, &restpackhtmltopdfapi.Scanner{}, &restpackscreenshotapi.Scanner{}, + &rev.Scanner{}, &revampcrm.Scanner{}, &ringcentral.Scanner{}, &ritekit.Scanner{}, @@ -1558,6 +1625,7 @@ func buildDetectorList() []detectors.Detector { &sheety.Scanner{}, &sherpadesk.Scanner{}, &shipday.Scanner{}, + &shippo.Scanner{}, &shodankey.Scanner{}, &shopify.Scanner{}, &shopifyoauth.Scanner{}, @@ -1587,10 +1655,13 @@ func buildDetectorList() []detectors.Detector { &snipcart.Scanner{}, &snowflake.Scanner{}, &snykkey.Scanner{}, - &sonarcloud.Scanner{}, + &solarwindsobservability.Scanner{}, + &sonarcloudv1.Scanner{}, + &sonarcloudv2.Scanner{}, &sourcegraph.Scanner{}, &sourcegraphcody.Scanner{}, // &sparkpost.Scanner{}, + &spectralops.Scanner{}, &speechtextai.Scanner{}, &splunkobservabilitytoken.Scanner{}, &spoonacular.Scanner{}, @@ -1654,6 +1725,7 @@ func buildDetectorList() []detectors.Detector { &timecamp.Scanner{}, &timezoneapi.Scanner{}, &tineswebhook.Scanner{}, + &tly.Scanner{}, &tmetric.Scanner{}, &todoist.Scanner{}, // &toggltrack.Scanner{}, @@ -1691,6 +1763,7 @@ func buildDetectorList() []detectors.Detector { &upwave.Scanner{}, &uri.Scanner{}, &urlscan.Scanner{}, + &user.Scanner{}, &userflow.Scanner{}, &userstack.Scanner{}, &vagrantcloudpersonaltoken.Scanner{}, @@ -1721,10 +1794,12 @@ func buildDetectorList() []detectors.Detector { &webscraper.Scanner{}, &webscraping.Scanner{}, &websitepulse.Scanner{}, - &weightsandbiases.Scanner{}, + &weightsandbiasesv1.Scanner{}, + &weightsandbiasesv2.Scanner{}, // &wepay.Scanner{}, &whoxy.Scanner{}, &wistia.Scanner{}, + &wit.Scanner{}, &wiz.Scanner{}, &worksnaps.Scanner{}, &workstack.Scanner{}, @@ -1754,6 +1829,87 @@ func buildDetectorList() []detectors.Detector { &zonkafeedback.Scanner{}, &zulipchat.Scanner{}, } + + dets = slices.DeleteFunc(dets, func(d detectors.Detector) bool { + switch d.(type) { + case *pinecone.Scanner: + return !feature.PineconeDetectorEnabled.Load() + case *cloudinary.Scanner: + return !feature.CloudinaryDetectorEnabled.Load() + case *gitlaboauth2.Scanner: + return !feature.GitLabOAuthDetectorEnabled.Load() + case *figmapersonalaccesstokenv3.Scanner: + return !feature.FigmaV3DetectorEnabled.Load() + case *sonarcloudv2.Scanner: + return !feature.SonarCloudV2DetectorEnabled.Load() + case *enigma.Scanner: + return !feature.EnigmaDetectorEnabled.Load() + case *datadogapikey.Scanner: + return !feature.DatadogApiKeyDetectorEnabled.Load() + case *tly.Scanner: + return !feature.TlyDetectorEnabled.Load() + case *wit.Scanner: + return !feature.WitDetectorEnabled.Load() + case *rev.Scanner: + return !feature.RevDetectorEnabled.Load() + case *user.Scanner: + return !feature.UserDetectorEnabled.Load() + case *braintrust.Scanner: + return !feature.BraintrustDetectorEnabled.Load() + case *pganalyzereadkey.Scanner: + return !feature.PgAnalyzeReadKeyDetectorEnabled.Load() + case *redhatpyxis.Scanner: + return !feature.RedHatPyxisDetectorEnabled.Load() + case *octopusdeploy.Scanner: + return !feature.OctopusDeployDetectorEnabled.Load() + case *openrouter.Scanner: + return !feature.OpenRouterDetectorEnabled.Load() + case *newrelicinsightsinsertkey.Scanner: + return !feature.NewRelicInsightsInsertKeyDetectorEnabled.Load() + case *duffeltoken.Scanner: + return !feature.DuffelTokenDetectorEnabled.Load() + case *shippo.Scanner: + return !feature.ShippoDetectorEnabled.Load() + case *ipinfo.Scanner: + return !feature.IPInfoDetectorEnabled.Load() + case *lob.Scanner: + return !feature.LobDetectorEnabled.Load() + case *hashicorpbatchtoken.Scanner: + return !feature.HashiCorpVaultBatchTokenDetectorEnabled.Load() + case *hashicorpvaulttoken.Scanner: + return !feature.HashiCorpVaultTokenDetectorEnabled.Load() + case *cloudflareapitokenv2.Scanner: + return !feature.CloudflareApiTokenV2DetectorEnabled.Load() + case *cloudflareglobalapikeyv2.Scanner: + return !feature.CloudflareGlobalApiKeyV2DetectorEnabled.Load() + case *duo.Scanner: + return !feature.DuoDetectorEnabled.Load() + case *newreliclicensekey.Scanner: + return !feature.NewRelicLicenseKeyDetectorEnabled.Load() + case *newrelicbrowserkey.Scanner: + return !feature.NewRelicBrowserKeyDetectorEnabled.Load() + case *newrelicuserkey.Scanner: + return !feature.NewRelicUserKeyDetectorEnabled.Load() + case *newrelicinsightsquerykey.Scanner: + return !feature.NewRelicInsightsQueryKeyDetectorEnabled.Load() + case *newrelicmobileapptoken.Scanner: + return !feature.NewRelicMobileAppTokenDetectorEnabled.Load() + case *microsoftteamswebhookv2.Scanner: + return !feature.MSTeamsWebhookV2DetectorEnabled.Load() + case *solarwindsobservability.Scanner: + return !feature.SolarwindsDetectorEnabled.Load() + case *humioapitoken.Scanner: + return !feature.HumioAPITokenDetectorEnabled.Load() + case *resend.Scanner: + return !feature.ResendDetectorEnabled.Load() + case *weightsandbiasesv2.Scanner: + return !feature.WeightsAndBiasesV2DetectorEnabled.Load() + default: + return false + } + }) + + return dets } func DefaultDetectors() []detectors.Detector { diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go index de3489b3e059..7155a22a1666 100644 --- a/pkg/engine/defaults/defaults_test.go +++ b/pkg/engine/defaults/defaults_test.go @@ -65,3 +65,306 @@ func TestDefaultVersionerDetectorsHaveNonZeroVersions(t *testing.T) { ) } } + +// TestAllDetectorTypesAreInDefaultList ensures every proto-defined DetectorType +// is either present in buildDetectorList() or explicitly excluded below. +// +// If you add a new DetectorType to the proto: +// - Add its scanner to buildDetectorList() in defaults.go, OR +// - Add it to excludedFromDefaultList in this test. +func TestAllDetectorTypesAreInDefaultList(t *testing.T) { + activeTypes := make(map[detector_typepb.DetectorType]struct{}) + for _, d := range DefaultDetectors() { + activeTypes[d.Type()] = struct{}{} + } + + for typeInt32, typeName := range detector_typepb.DetectorType_name { + dt := detector_typepb.DetectorType(typeInt32) + if _, ok := activeTypes[dt]; ok { + continue + } + if _, ok := excludedFromDefaultList[dt]; ok { + continue + } + t.Errorf( + "DetectorType %q (value %d) is missing from buildDetectorList(); "+ + "add it to defaults.go or to excludedFromDefaultList in this test", + typeName, typeInt32, + ) + } + + // Reverse check: no excluded type should appear in the active list. + // This catches types that were added to the exclude list by mistake. + for dt := range excludedFromDefaultList { + if _, ok := activeTypes[dt]; ok { + t.Errorf( + "DetectorType %q is in excludedFromDefaultList but is also present in buildDetectorList(); "+ + "remove it from excludedFromDefaultList", + detector_typepb.DetectorType_name[int32(dt)], + ) + } + } +} + +// excludedFromDefaultList contains detector types that are intentionally absent +// from buildDetectorList(). Keep entries grouped and sorted. +// +// TODO: audit this list periodically — entries in the "mistakenly missed" group +// should be removed once the corresponding detector is added to defaults.go. +// +//nolint:staticcheck // SA1019: intentionally references deprecated DetectorType values to keep them excluded. +var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ + // TODO: these detectors have implementations but were mistakenly never added + // to buildDetectorList() — discovered by TestAllDetectorTypesAreInDefaultList. + // They are not added immediately out of caution for the impact on customers/users. + // Remove each entry once its detector has been carefully added. + detector_typepb.DetectorType_Guru: {}, + detector_typepb.DetectorType_Tru: {}, + + // Feature flag gated detectors + // These should be removed from this list when we remove the feature flag + detector_typepb.DetectorType_Cloudinary: {}, + detector_typepb.DetectorType_DatadogApikey: {}, + detector_typepb.DetectorType_Enigma: {}, + detector_typepb.DetectorType_GitLabOauth2: {}, + detector_typepb.DetectorType_Pinecone: {}, + detector_typepb.DetectorType_TLy: {}, + detector_typepb.DetectorType_Wit: {}, + detector_typepb.DetectorType_Rev: {}, + detector_typepb.DetectorType_User: {}, + detector_typepb.DetectorType_BrainTrustApiKey: {}, + detector_typepb.DetectorType_PgAnalyzeReadKey: {}, + detector_typepb.DetectorType_RedHatPyxis: {}, + detector_typepb.DetectorType_OctopusDeploy: {}, + detector_typepb.DetectorType_OpenRouter: {}, + detector_typepb.DetectorType_NewRelicInsightsInsertKey: {}, + detector_typepb.DetectorType_DuffelToken: {}, + detector_typepb.DetectorType_Shippo: {}, + detector_typepb.DetectorType_IPInfo: {}, + detector_typepb.DetectorType_Lob: {}, + detector_typepb.DetectorType_HashiCorpVaultBatchToken: {}, + detector_typepb.DetectorType_HashiCorpVaultToken: {}, + detector_typepb.DetectorType_Duo: {}, + detector_typepb.DetectorType_NewRelicLicenseKey: {}, + detector_typepb.DetectorType_NewRelicBrowserKey: {}, + detector_typepb.DetectorType_NewRelicUserKey: {}, + detector_typepb.DetectorType_NewRelicInsightsQueryKey: {}, + detector_typepb.DetectorType_NewRelicMobileAppToken: {}, + detector_typepb.DetectorType_SolarWindsObservability: {}, + detector_typepb.DetectorType_HumioAPIToken: {}, + detector_typepb.DetectorType_Resend: {}, + + // Reserved / special types. + detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList() + detector_typepb.DetectorType_Test: {}, + + // Deprecated — the proto field is marked deprecated=true and the service + // no longer exists or has been superseded by a different type. + detector_typepb.DetectorType_AirtableApiKey: {}, + detector_typepb.DetectorType_ApiScience: {}, + detector_typepb.DetectorType_AppOptics: {}, + detector_typepb.DetectorType_BingSubscriptionKey: {}, + detector_typepb.DetectorType_Blablabus: {}, + detector_typepb.DetectorType_CoinbaseWaaS: {}, + detector_typepb.DetectorType_CoinMarketCap: {}, + detector_typepb.DetectorType_CrossBrowserTesting: {}, + detector_typepb.DetectorType_DataFire: {}, + detector_typepb.DetectorType_EtsyApiKey: {}, + detector_typepb.DetectorType_FakeJSON: {}, + detector_typepb.DetectorType_Flowdash: {}, + detector_typepb.DetectorType_Flowdock: {}, + detector_typepb.DetectorType_Fusebill: {}, + detector_typepb.DetectorType_GlitterlyAPI: {}, + detector_typepb.DetectorType_GoogleApiKey: {}, + detector_typepb.DetectorType_Happi: {}, + detector_typepb.DetectorType_Heatmapapi: {}, + detector_typepb.DetectorType_Integromat: {}, + detector_typepb.DetectorType_Ipify: {}, + detector_typepb.DetectorType_Lastfm: {}, + detector_typepb.DetectorType_Macaddress: {}, + detector_typepb.DetectorType_Nitro: {}, + detector_typepb.DetectorType_Nytimes: {}, + detector_typepb.DetectorType_OnWaterIO: {}, + detector_typepb.DetectorType_Opengraphr: {}, + detector_typepb.DetectorType_Passbase: {}, + detector_typepb.DetectorType_ProspectIO: {}, + detector_typepb.DetectorType_QuickMetrics: {}, + detector_typepb.DetectorType_Restpack: {}, + detector_typepb.DetectorType_Rockset: {}, + detector_typepb.DetectorType_ScraperSite: {}, + detector_typepb.DetectorType_Sentiment: {}, + detector_typepb.DetectorType_SportRadar: {}, + detector_typepb.DetectorType_Squareup: {}, + detector_typepb.DetectorType_Text2Data: {}, + + // Intentionally commented out in buildDetectorList() — implementation + // exists but the detector is disabled (API issues, false positives, etc.). + detector_typepb.DetectorType_Abstract: {}, + detector_typepb.DetectorType_AdobeIO: {}, + detector_typepb.DetectorType_Alconost: {}, + detector_typepb.DetectorType_Apollo: {}, + detector_typepb.DetectorType_AzureFunctionKey: {}, + detector_typepb.DetectorType_Besnappy: {}, + detector_typepb.DetectorType_BlockNative: {}, + detector_typepb.DetectorType_DailyCO: {}, + detector_typepb.DetectorType_Debounce: {}, + detector_typepb.DetectorType_Generic: {}, + detector_typepb.DetectorType_GetEmail: {}, + detector_typepb.DetectorType_GetEmails: {}, + detector_typepb.DetectorType_Hive: {}, + detector_typepb.DetectorType_IbmCloudUserKey: {}, + detector_typepb.DetectorType_LineMessaging: {}, + detector_typepb.DetectorType_M3o: {}, + detector_typepb.DetectorType_Magnetic: {}, + detector_typepb.DetectorType_Manifest: {}, + detector_typepb.DetectorType_Mixpanel: {}, + detector_typepb.DetectorType_NasdaqDataLink: {}, + detector_typepb.DetectorType_Raven: {}, + detector_typepb.DetectorType_Sparkpost: {}, + detector_typepb.DetectorType_SpotifyKey: {}, + detector_typepb.DetectorType_TogglTrack: {}, + detector_typepb.DetectorType_WePay: {}, + detector_typepb.DetectorType_ZapierWebhook: {}, + + // Not yet implemented — proto enum entry exists but no detector has been + // written for this service yet. + detector_typepb.DetectorType_Aerisweather: {}, + detector_typepb.DetectorType_Aftership: {}, + detector_typepb.DetectorType_Convert: {}, + detector_typepb.DetectorType_Honey: {}, + detector_typepb.DetectorType_HubSpot: {}, + detector_typepb.DetectorType_Paymo: {}, + detector_typepb.DetectorType_Sellfy: {}, + detector_typepb.DetectorType_AirtableMetadataApiKey: {}, + detector_typepb.DetectorType_AkamaiToken: {}, + detector_typepb.DetectorType_Alphavantage: {}, + detector_typepb.DetectorType_AmazonMWS: {}, + detector_typepb.DetectorType_AMQP: {}, + detector_typepb.DetectorType_Api2Convert: {}, + detector_typepb.DetectorType_Authorize: {}, + detector_typepb.DetectorType_Avalara: {}, + detector_typepb.DetectorType_BaseApiIO: {}, + detector_typepb.DetectorType_BasisTheory: {}, + detector_typepb.DetectorType_BitGo: {}, + detector_typepb.DetectorType_Bored: {}, + detector_typepb.DetectorType_Brightlocal: {}, + detector_typepb.DetectorType_Bubble: {}, + detector_typepb.DetectorType_Checkmarket: {}, + detector_typepb.DetectorType_CircleCI: {}, + detector_typepb.DetectorType_Cloudant: {}, + detector_typepb.DetectorType_CloudsightKey: {}, + detector_typepb.DetectorType_Cloudways: {}, + detector_typepb.DetectorType_CoinGecko: {}, + detector_typepb.DetectorType_Cometchat: {}, + detector_typepb.DetectorType_ContentfulDelivery: {}, + detector_typepb.DetectorType_ContentStack: {}, + detector_typepb.DetectorType_Copyscape: {}, + detector_typepb.DetectorType_Createsend: {}, + detector_typepb.DetectorType_Cricket: {}, + detector_typepb.DetectorType_DigitalOceanSpaces: {}, + detector_typepb.DetectorType_Distribusion: {}, //nolint:misspell // proto enum name is intentionally spelled this way + detector_typepb.DetectorType_Duda: {}, + detector_typepb.DetectorType_Duffel: {}, + detector_typepb.DetectorType_Dynadot: {}, + detector_typepb.DetectorType_Dynatrace: {}, + detector_typepb.DetectorType_Edusign: {}, + detector_typepb.DetectorType_ElasticPath: {}, + detector_typepb.DetectorType_Emailoctopus: {}, + detector_typepb.DetectorType_EquinixOauth: {}, + detector_typepb.DetectorType_Eversign: {}, + detector_typepb.DetectorType_Fastspring: {}, + detector_typepb.DetectorType_Feedly: {}, + detector_typepb.DetectorType_Filestack: {}, + detector_typepb.DetectorType_Firebase: {}, + detector_typepb.DetectorType_FirebaseCloudMessaging: {}, + detector_typepb.DetectorType_FlagsmithEnvironmentKey: {}, + detector_typepb.DetectorType_FlagsmithToken: {}, + detector_typepb.DetectorType_Formstack: {}, + detector_typepb.DetectorType_Fountain: {}, + detector_typepb.DetectorType_FullContact: {}, + detector_typepb.DetectorType_GitHubOld: {}, + detector_typepb.DetectorType_Goshippo: {}, + detector_typepb.DetectorType_Gosquared: {}, + detector_typepb.DetectorType_Hotwire: {}, + detector_typepb.DetectorType_HubSpotOauth: {}, + detector_typepb.DetectorType_HypeAuditor: {}, + detector_typepb.DetectorType_Image4: {}, + detector_typepb.DetectorType_ImageToText: {}, + detector_typepb.DetectorType_Imgix: {}, + detector_typepb.DetectorType_Imgur: {}, + detector_typepb.DetectorType_Infobip: {}, + detector_typepb.DetectorType_JSONbin: {}, + detector_typepb.DetectorType_Jumpseller: {}, + detector_typepb.DetectorType_Kairos: {}, + detector_typepb.DetectorType_KakaoTalk: {}, + detector_typepb.DetectorType_Kaleyra: {}, + detector_typepb.DetectorType_KalturaAppToken: {}, + detector_typepb.DetectorType_KalturaSession: {}, + detector_typepb.DetectorType_Keygen: {}, + detector_typepb.DetectorType_KiteConnect: {}, + detector_typepb.DetectorType_KubeConfig: {}, + detector_typepb.DetectorType_LinkedIn: {}, + detector_typepb.DetectorType_Linode: {}, + detector_typepb.DetectorType_Messari: {}, + detector_typepb.DetectorType_Midise: {}, + detector_typepb.DetectorType_Mixcloud: {}, + detector_typepb.DetectorType_Mojohelpdesk: {}, + detector_typepb.DetectorType_MollieAccessToken: {}, + detector_typepb.DetectorType_MollieAPIKey: {}, + detector_typepb.DetectorType_Myexperiment: {}, + detector_typepb.DetectorType_NetCore: {}, + detector_typepb.DetectorType_NiceHash: {}, + detector_typepb.DetectorType_Nordigen: {}, + detector_typepb.DetectorType_Nubela: {}, + detector_typepb.DetectorType_OcrSpace: {}, + detector_typepb.DetectorType_Onbuka: {}, + detector_typepb.DetectorType_Opendatasoft: {}, + detector_typepb.DetectorType_Optidash: {}, + detector_typepb.DetectorType_Paddle: {}, + detector_typepb.DetectorType_Page2Images: {}, + detector_typepb.DetectorType_Papyrs: {}, + detector_typepb.DetectorType_PDFmyURL: {}, + detector_typepb.DetectorType_PendoIntegrationKey: {}, + detector_typepb.DetectorType_PlaidToken: {}, + detector_typepb.DetectorType_Printfection: {}, + detector_typepb.DetectorType_Processst: {}, + detector_typepb.DetectorType_Quickbase: {}, + detector_typepb.DetectorType_ReCAPTCHA: {}, + detector_typepb.DetectorType_Redbooth: {}, + detector_typepb.DetectorType_Riotgames: {}, + detector_typepb.DetectorType_Rosette: {}, + detector_typepb.DetectorType_Samsara: {}, + detector_typepb.DetectorType_ScrapingDog: {}, + detector_typepb.DetectorType_Sendoso: {}, + detector_typepb.DetectorType_ShopeeOpenPlatform: {}, + detector_typepb.DetectorType_Simplybook: {}, + detector_typepb.DetectorType_SMSApi: {}, + detector_typepb.DetectorType_StreamChatMessaging: {}, + detector_typepb.DetectorType_Supportbee: {}, + detector_typepb.DetectorType_Surge: {}, + detector_typepb.DetectorType_Teamup: {}, + detector_typepb.DetectorType_TeamViewer: {}, + detector_typepb.DetectorType_Telesign: {}, + detector_typepb.DetectorType_TencentCloudKey: {}, + detector_typepb.DetectorType_Timekit: {}, + detector_typepb.DetectorType_Trimble: {}, + detector_typepb.DetectorType_TwitterApiSecret: {}, + detector_typepb.DetectorType_UberServerToken: {}, + detector_typepb.DetectorType_Uproc: {}, + detector_typepb.DetectorType_Veevavault: {}, + detector_typepb.DetectorType_Vonage: {}, + detector_typepb.DetectorType_Wakatime: {}, + detector_typepb.DetectorType_Webengage: {}, + detector_typepb.DetectorType_WeChatAppKey: {}, + detector_typepb.DetectorType_Woopra: {}, + detector_typepb.DetectorType_WordsApi: {}, + detector_typepb.DetectorType_Workday: {}, + detector_typepb.DetectorType_WpEngine: {}, + detector_typepb.DetectorType_Yext: {}, + detector_typepb.DetectorType_AzureActiveDirectoryApplicationSecret: {}, + detector_typepb.DetectorType_AzureCacheForRedisAccessKey: {}, + detector_typepb.DetectorType_AzureManagementCertificate: {}, + detector_typepb.DetectorType_AzureMLWebServiceClassicIdentifiableKey: {}, + detector_typepb.DetectorType_AzureSQL: {}, + detector_typepb.DetectorType_BuiltWith: {}, +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 180e9e18900c..408496a8049e 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -5,6 +5,7 @@ package engine import ( "bytes" + "crypto/md5" "errors" "fmt" "runtime" @@ -39,8 +40,8 @@ import ( var detectionTimeout = detectors.DefaultResponseTimeout var errOverlap = errors.New( - "More than one detector has found this result. For your safety, verification has been disabled." + - "You can override this behavior by using the --allow-verification-overlap flag.", + "more than one detector has found this result; for your safety, verification has been disabled. " + + "You can override this behavior by using the --allow-verification-overlap flag", ) // Metrics for the scan engine for external consumption. @@ -125,10 +126,17 @@ type Config struct { FilterUnverified bool ShouldScanEntireChunk bool + // NoIgnoreTag disables the "trufflehog:ignore" tag. If set to true, results are + // reported even when the line they were found on carries the tag. + NoIgnoreTag bool + Dispatcher ResultsDispatcher - // SourceManager is used to manage the sources and units. - // TODO (ahrav): Update this comment, i'm dumb and don't really know what else it does. + // SourceManager orchestrates source enumeration and concurrent chunk production. + // The engine consumes chunks from its output channel via ChunksChan, and during + // Finish it calls SourceManager.Wait() to drain in-flight work before closing the + // downstream channels. Callers construct and own the SourceManager; the engine + // neither creates nor closes it. SourceManager *sources.SourceManager // PrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to @@ -191,6 +199,8 @@ type Engine struct { // By default, the engine will only scan a subset of the chunk if a detector matches the chunk. // If this flag is set to true, the engine will scan the entire chunk. scanEntireChunk bool + // noIgnoreTag disables the "trufflehog:ignore" tag, so tagged lines are still reported. + noIgnoreTag bool // ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups. AhoCorasickCore *ahocorasick.Core @@ -215,7 +225,7 @@ type Engine struct { // dedupeCache is used to deduplicate results by comparing the // detector type, raw result, and source metadata - dedupeCache *lru.Cache[string, detectorspb.DecoderType] + dedupeCache *lru.Cache[string, struct{}] // verify determines whether the scanner will attempt to verify candidate secrets. verify bool @@ -231,6 +241,10 @@ type Engine struct { verificationOverlapWorkerMultiplier int maxDecodeDepth int + + // runtimeCollector exposes live channel/worker/scan counters to Prometheus + // while the engine is running. Set in Start, cleared in Finish. + runtimeCollector *runtimeCollector } // NewEngine creates a new Engine instance with the provided configuration. @@ -251,6 +265,7 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) { verificationOverlap: cfg.VerificationOverlap, sourceManager: cfg.SourceManager, scanEntireChunk: cfg.ShouldScanEntireChunk, + noIgnoreTag: cfg.NoIgnoreTag, detectorVerificationOverrides: cfg.DetectorVerificationOverrides, detectorWorkerMultiplier: cfg.DetectorWorkerMultiplier, notificationWorkerMultiplier: cfg.NotificationWorkerMultiplier, @@ -503,10 +518,11 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors // deduplication efforts, allowing the engine to quickly check if a chunk has // been processed before, thereby saving computational overhead. func (e *Engine) initialize(ctx context.Context) error { - // TODO (ahrav): Determine the optimal cache size. - const cacheSize = 512 // number of entries in the LRU cache + // The cache size is set to 5000 entries, which is a balance between memory usage and the need for effective deduplication. + // Since the cache entries are md5 hashes so each entry would be 16 bytes, so in total this would be aorund 80KB of memory usage. + const cacheSize = 5000 - cache, err := lru.New[string, detectorspb.DecoderType](cacheSize) + cache, err := lru.New[string, struct{}](cacheSize) if err != nil { return fmt.Errorf("failed to initialize LRU cache: %w", err) } @@ -637,6 +653,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration { func (e *Engine) Start(ctx context.Context) { e.metrics = runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}} e.sanityChecks(ctx) + e.registerRuntimeMetrics(ctx) e.startWorkers(ctx) } @@ -754,6 +771,8 @@ func (e *Engine) Finish(ctx context.Context) error { e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime) + e.unregisterRuntimeMetrics() + return err } @@ -829,11 +848,11 @@ func iterativeDecode(chunk *sources.Chunk, allDecoders []decoders.Decoder, maxDe if depth+1 < maxDepth && decoder.Type() != detectorspb.DecoderType_PLAIN && - !bytes.Equal(decoded.Chunk.Data, data) && - !slices.ContainsFunc(seen, func(s []byte) bool { return bytes.Equal(s, decoded.Chunk.Data) }) { + !bytes.Equal(decoded.Data, data) && + !slices.ContainsFunc(seen, func(s []byte) bool { return bytes.Equal(s, decoded.Data) }) { - seen = append(seen, decoded.Chunk.Data) - nextInputs = append(nextInputs, decoded.Chunk.Data) + seen = append(seen, decoded.Data) + nextInputs = append(nextInputs, decoded.Data) } } } @@ -854,12 +873,18 @@ func (e *Engine) scannerWorker(ctx context.Context) { for chunk := range e.ChunksChan() { startTime := time.Now() sourceVerify := chunk.SourceVerify + sourceTypeStr := chunk.SourceType.String() + chunksEnteredStage.WithLabelValues("scanner", sourceTypeStr).Inc() chunk.OriginalData = chunk.Data decoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth) for _, d := range decoded { - matchingDetectors := e.AhoCorasickCore.FindDetectorMatches(d.Chunk.Data) + matchingDetectors := e.AhoCorasickCore.FindDetectorMatches(d.Data) + if len(matchingDetectors) == 0 { + chunksDropped.WithLabelValues("scanner", "no_matching_detectors", sourceTypeStr).Inc() + continue + } if len(matchingDetectors) > 1 && !e.verificationOverlap { wgVerificationOverlap.Add(1) e.verificationOverlapChunksChan <- verificationOverlapChunk{ @@ -995,8 +1020,11 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) { chunkSecrets := make(map[chunkSecretKey]struct{}, avgSecretsPerDetector) for chunk := range e.verificationOverlapChunksChan { + sourceTypeStr := chunk.chunk.SourceType.String() + chunksEnteredStage.WithLabelValues("verification_overlap", sourceTypeStr).Inc() for _, detector := range chunk.detectors { isFalsePositive := detectors.GetFalsePositiveCheck(detector.Detector) + detectorNameStr := detector.Key.Type().String() // DO NOT VERIFY at this stage of the pipeline. matchedBytes := detector.Matches() @@ -1007,8 +1035,9 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) { if err != nil { ctx.Logger().Error( err, "error finding results in chunk during verification overlap", - "detector", detector.Key.Type().String(), + "detector", detectorNameStr, ) + chunksDropped.WithLabelValues("verification_overlap", "detector_error", sourceTypeStr).Inc() } if len(results) == 0 { @@ -1024,7 +1053,7 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) { // disable filtration for targeted scans, but if you're here because this problem surfaced for a // non-targeted scan then we'll have to solve it correctly. if chunk.chunk.SecretID == 0 { - results = e.filterResults(ctx, detector, results) + results = e.filterResults(ctx, "verification_overlap", detector, results) } for _, res := range results { @@ -1041,6 +1070,7 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) { // - malicious detector "api key": qnwfsLyRSyfCwfpHaQP1UzDhrgpWvHjbYzjpRCMshjt417zWcrzyHUArs7r key := chunkSecretKey{secret: string(val), detectorKey: detector.Key} if _, ok := chunkSecrets[key]; ok { + resultsDropped.WithLabelValues("verification_overlap", "intra_chunk_duplicate", detectorNameStr).Inc() continue } @@ -1056,7 +1086,7 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) { res, chunk.chunk, chunk.decoder, - detector.Detector.Description(), + detector.Description(), isFalsePositive, ) @@ -1119,6 +1149,10 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) { ctx.Logger().V(5).Info("Starting to detect chunk") + sourceTypeStr := data.chunk.SourceType.String() + detectorNameStr := data.detector.Type().String() + chunksEnteredStage.WithLabelValues("detect", sourceTypeStr).Inc() + isFalsePositive := detectors.GetFalsePositiveCheck(data.detector.Detector) var matchCount int @@ -1146,16 +1180,21 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) { cancel() if err != nil { ctx.Logger().Error(err, "error finding results in chunk") + chunksDropped.WithLabelValues("detect", "detector_error", sourceTypeStr).Inc() continue } + if len(results) > 0 { + resultsProduced.WithLabelValues(detectorNameStr).Add(float64(len(results))) + } + detectorExecutionCount.WithLabelValues( - data.detector.Type().String(), - strconv.Itoa(int(data.chunk.JobID)), + detectorNameStr, + sourceTypeStr, data.chunk.SourceName, ).Inc() detectorExecutionDuration.WithLabelValues( - data.detector.Type().String(), + detectorNameStr, ).Observe(float64(time.Since(start).Milliseconds())) if e.printAvgDetectorTime && len(results) > 0 { @@ -1179,11 +1218,13 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) { // scans, but if you're here because this problem surfaced for a non-targeted scan then we'll have to solve it // correctly. if data.chunk.SecretID == 0 { - results = e.filterResults(ctx, data.detector, results) + results = e.filterResults(ctx, "detect", data.detector, results) } + AssignDuplicateLineOffsets(&data.chunk, results) + for _, res := range results { - e.processResult(ctx, res, data.chunk, data.decoder, data.detector.Detector.Description(), isFalsePositive) + e.processResult(ctx, res, data.chunk, data.decoder, data.detector.Description(), isFalsePositive) } } @@ -1196,9 +1237,11 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) { func (e *Engine) filterResults( ctx context.Context, + stage string, detector *ahocorasick.DetectorMatch, results []detectors.Result, ) []detectors.Result { + detectorNameStr := detector.Type().String() clean := detectors.CleanResults ignoreConfig := false if cleaner, ok := detector.Detector.(detectors.CustomResultsCleaner); ok { @@ -1206,18 +1249,31 @@ func (e *Engine) filterResults( ignoreConfig = cleaner.ShouldCleanResultsIrrespectiveOfConfiguration() } if e.filterUnverified || ignoreConfig { - results = clean(results) + reason := "filter_unverified" + if !e.filterUnverified && ignoreConfig { + reason = "custom_cleaner" + } + before := len(results) + results = clean(results, e.verify) + if dropped := before - len(results); dropped > 0 { + resultsDropped.WithLabelValues(stage, reason, detectorNameStr).Add(float64(dropped)) + } } if e.filterEntropy != 0 { + before := len(results) results = detectors.FilterResultsWithEntropy(ctx, results, e.filterEntropy, e.retainFalsePositives) + if dropped := before - len(results); dropped > 0 { + resultsDropped.WithLabelValues(stage, "filter_entropy", detectorNameStr).Add(float64(dropped)) + } } return results } // processResult generates a detectors.ResultWithMetadata from the provided chunk and result and puts it on the results -// channel, unless the result exists on a line with an ignore tag, in which case no result is generated. +// channel, unless the result exists on a line with an ignore tag and --no-ignore-tag is not passed, in which case +// no result is generated. func (e *Engine) processResult( ctx context.Context, res detectors.Result, @@ -1237,11 +1293,13 @@ func (e *Engine) processResult( ignoreLinePresent = SetResultLineNumber(©Chunk, &res, fragStart, mdLine) if err := UpdateLink(ctx, copyChunk.SourceMetadata, link, *mdLine); err != nil { ctx.Logger().Error(err, "error setting link") + resultsDropped.WithLabelValues("process_result", "update_link_error", res.DetectorType.String()).Inc() return } chunk = copyChunk } - if ignoreLinePresent { + if ignoreLinePresent && !e.noIgnoreTag { + resultsDropped.WithLabelValues("process_result", "ignore_line_tag", res.DetectorType.String()).Inc() return } @@ -1260,39 +1318,55 @@ func (e *Engine) processResult( func (e *Engine) notifierWorker(ctx context.Context) { for result := range e.ResultsChan() { startTime := time.Now() + detectorNameStr := result.DetectorType.String() // Filter unwanted results, based on `--results`. if !result.Verified { if result.IsWordlistFalsePositive && !e.retainFalsePositives { // Skip false positives + resultsDropped.WithLabelValues("notifier", "wordlist_false_positive", detectorNameStr).Inc() continue } else if result.VerificationError() != nil { if !e.notifyUnknownResults { // Skip results with verification errors. + resultsDropped.WithLabelValues("notifier", "unknown_result_filtered", detectorNameStr).Inc() continue } } else if !e.notifyUnverifiedResults { // Skip unverified results. + resultsDropped.WithLabelValues("notifier", "unverified_result_filtered", detectorNameStr).Inc() continue } } else if !e.notifyVerifiedResults { // Skip verified results. // TODO: Is this a legitimate use case? + resultsDropped.WithLabelValues("notifier", "verified_result_filtered", detectorNameStr).Inc() continue } atomic.AddUint32(&e.numFoundResults, 1) // Dedupe results by comparing the detector type, raw result, and source metadata. - // We want to avoid duplicate results with different decoder types, but we also - // want to include duplicate results with the same decoder type. - // Duplicate results with the same decoder type SHOULD have their own entry in the - // results list, this would happen if the same secret is found multiple times. - // Note: If the source type is postman, we dedupe the results regardless of decoder type. - key := fmt.Sprintf("%s%s%s%+v", result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata) - if val, ok := e.dedupeCache.Get(key); ok && (val != result.DecoderType || - result.SourceType == sourcespb.SourceType_SOURCE_TYPE_POSTMAN) { - continue + // The key includes SourceMetadata (file path, line numbers, etc.) so genuinely + // different occurrences of the same credential at different locations are not + // suppressed. Only exact duplicates — same credential at the same location — are + // filtered, which handles peek-overlap re-scans and cross-decoder duplicates alike. + // The key also include the DetectorName to prevent deduping the results for + // custom detectors which have the same type. + // MD5 hash of the key is used to reduce memory usage of the dedupe cache, + // since the raw result and source metadata can be large. + // + // This deduplication only applies to results that are *not* + // from reverification, since we are expected to see the same + // result from reverification and want to Dispatch it below. + + // Notifier workers share this cache; the check and insert must be one atomic step. + if result.SecretID == 0 { + h := md5.Sum([]byte(fmt.Sprintf("%s%s%s%s%+v", result.DetectorName, result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata))) + key := string(h[:]) + if found, _ := e.dedupeCache.ContainsOrAdd(key, struct{}{}); found { + resultsDropped.WithLabelValues("notifier", "dedupe_cache_hit", detectorNameStr).Inc() + continue + } } - e.dedupeCache.Add(key, result.DecoderType) if result.Verified { atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1) @@ -1302,6 +1376,9 @@ func (e *Engine) notifierWorker(ctx context.Context) { if err := e.dispatcher.Dispatch(ctx, result); err != nil { ctx.Logger().Error(err, "error notifying result") + resultsDispatched.WithLabelValues(detectorNameStr, strconv.FormatBool(result.Verified), "false").Inc() + } else { + resultsDispatched.WithLabelValues(detectorNameStr, strconv.FormatBool(result.Verified), "true").Inc() } chunksNotifiedLatency.Observe(float64(time.Since(startTime).Milliseconds())) @@ -1327,21 +1404,44 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool { } } +// effectiveSecret returns the canonical secret string for a result, preferring +// the primary secret value and falling back to Raw. Callers that compute or +// consume chunk offsets must go through this helper so the two stay in sync. +func effectiveSecret(r *detectors.Result) string { + secret := r.GetPrimarySecretValue() + if secret == "" { + secret = string(r.Raw) + } + return secret +} + // FragmentLineOffset sets the line number for a provided source chunk with a given detector result. func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) { - // get the primary secret value from the result if set - secret := result.GetPrimarySecretValue() - if secret == "" { - secret = string(result.Raw) + secretBytes := []byte(effectiveSecret(result)) + + // Locate the byte offset of the secret in chunk.Data. If a chunk offset was + // pre-assigned (for duplicate secrets), use it directly to find the correct + // occurrence instead of always matching the first one. + var offset int + if result.HasChunkOffset() { + offset = int(result.ChunkOffset()) + } else { + offset = bytes.Index(chunk.Data, secretBytes) + if offset == -1 { + return 0, false + } } - before, after, found := bytes.Cut(chunk.Data, []byte(secret)) - if !found { - return 0, false + data := chunk.Data + if originalOffset := sourceOffset(chunk.OriginalData, chunk.Data, offset, len(secretBytes)); originalOffset >= 0 { + data, offset = chunk.OriginalData, originalOffset } - lineNumber := int64(bytes.Count(before, []byte("\n"))) + + lineNumber := int64(bytes.Count(data[:offset], []byte("\n"))) result.SetPrimarySecretLine(lineNumber) - // If the line contains the ignore tag, we should ignore the result. + + // If the line containing the secret has the ignore tag, we should ignore the result. + after := data[offset+len(secretBytes):] endLine := bytes.Index(after, []byte("\n")) if endLine == -1 { endLine = len(after) @@ -1352,6 +1452,164 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, return lineNumber, false } +// sourceOffset maps an offset in decoded data back onto the pre-decode buffer, +// returning -1 when the detected value has no counterpart in the source. +func sourceOffset(originalData, data []byte, offset, length int) int { + if len(originalData) == 0 || length == 0 || offset < 0 || offset+length > len(data) { + return -1 + } + if bytes.Equal(originalData, data) { + return offset + } + + secret := data[offset : offset+length] + preceding := bytes.Count(data[:offset], secret) + sourceIndex, sourceCount := nthOccurrence(originalData, secret, preceding) + if sourceCount == 0 { + return -1 + } + // Decoders emit the occurrences they keep in source order, so an unchanged + // occurrence count makes the nth decoded match the nth source match. + if sourceCount == preceding+bytes.Count(data[offset:], secret) { + return sourceIndex + } + // Decoding dropped or merged occurrences, so order alone no longer identifies + // the match and the surrounding text has to break the tie. + return bestAlignedOccurrence(originalData, data, offset, length) +} + +// nthOccurrence returns the offset of the nth zero-indexed non-overlapping +// occurrence of sep in data along with the total occurrence count. The offset is +// -1 when data holds fewer than n+1 occurrences. +func nthOccurrence(data, sep []byte, n int) (int, int) { + index, count, start := -1, 0, 0 + for { + next := bytes.Index(data[start:], sep) + if next == -1 { + return index, count + } + if count == n { + index = start + next + } + count++ + start += next + len(sep) + } +} + +// alignedContextBytes bounds the neighbourhood each candidate source occurrence is +// scored over, keeping the comparison linear in the number of candidates. A kilobyte +// is far more context than a decoder needs to give itself away. +const alignedContextBytes = 1024 + +// bestAlignedOccurrence picks the source occurrence of the detected value whose +// neighbourhood best survives into the decoded neighbourhood. No rule is exact here: +// when a decoder drops one copy of a value and keeps another, the copies are only +// distinguishable by the text around them. +func bestAlignedOccurrence(originalData, data []byte, offset, length int) int { + secret := data[offset : offset+length] + best, bestScore := -1, -1 + for start := 0; ; { + next := bytes.Index(originalData[start:], secret) + if next == -1 { + return best + } + candidate := start + next + if score := alignmentScore(originalData, data, candidate, offset, length); score > bestScore { + best, bestScore = candidate, score + } + start = candidate + length + } +} + +// alignmentScore measures how much of the decoded neighbourhood still reads, in +// order, out of the source around a candidate. Decoders interleave removals with the +// text they keep, so the source side is allowed gaps that the decoded side is not. +func alignmentScore(originalData, data []byte, candidate, offset, length int) int { + sourceBefore := lastBytes(originalData[:candidate], alignedContextBytes) + decodedBefore := lastBytes(data[:offset], alignedContextBytes) + sourceAfter := firstBytes(originalData[candidate+length:], alignedContextBytes) + decodedAfter := firstBytes(data[offset+length:], alignedContextBytes) + return matchBackward(sourceBefore, decodedBefore) + matchForward(sourceAfter, decodedAfter) +} + +// matchBackward returns the length of the longest suffix of decoded that appears as a +// subsequence of source. Matching greedily from the right is optimal for that. +func matchBackward(source, decoded []byte) int { + matched, j := 0, len(decoded)-1 + for i := len(source) - 1; i >= 0 && j >= 0; i-- { + if source[i] == decoded[j] { + matched, j = matched+1, j-1 + } + } + return matched +} + +// matchForward returns the length of the longest prefix of decoded that appears as a +// subsequence of source. +func matchForward(source, decoded []byte) int { + matched := 0 + for i := 0; i < len(source) && matched < len(decoded); i++ { + if source[i] == decoded[matched] { + matched++ + } + } + return matched +} + +func lastBytes(data []byte, n int) []byte { + if len(data) > n { + return data[len(data)-n:] + } + return data +} + +func firstBytes(data []byte, n int) []byte { + if len(data) > n { + return data[:n] + } + return data +} + +// AssignDuplicateLineOffsets pre-computes byte offsets for results that share the same +// secret value within a chunk. This allows FragmentLineOffset to locate the correct +// occurrence instead of always finding the first one. +func AssignDuplicateLineOffsets(chunk *sources.Chunk, results []detectors.Result) { + // Group result indices by their secret value. + type group struct { + secret string + indices []int + } + seen := make(map[string]int) // secret -> index into groups slice + var groups []group + + for i := range results { + secret := effectiveSecret(&results[i]) + if idx, ok := seen[secret]; ok { + groups[idx].indices = append(groups[idx].indices, i) + } else { + seen[secret] = len(groups) + groups = append(groups, group{secret: secret, indices: []int{i}}) + } + } + + for _, g := range groups { + if len(g.indices) <= 1 { + continue + } + secretBytes := []byte(g.secret) + searchStart := 0 + for _, ri := range g.indices { + pos := bytes.Index(chunk.Data[searchStart:], secretBytes) + if pos == -1 { + break + } + absPos := searchStart + pos + results[ri].SetChunkOffset(int64(absPos)) + searchStart = absPos + len(secretBytes) + } + } +} + // FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata. // It returns: // - The first line number of the fragment. diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index e5ca7ef95733..c7c6419ac1a8 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -1,6 +1,7 @@ package engine import ( + "bytes" aCtx "context" "fmt" "math/rand" @@ -8,14 +9,17 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync" "sync/atomic" "testing" "time" "github.com/google/go-cmp/cmp" + lru "github.com/hashicorp/golang-lru/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" + "pgregory.net/rapid" "github.com/trufflesecurity/trufflehog/v3/pkg/config" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -231,6 +235,292 @@ func TestFragmentLineOffsetWithPrimarySecretMultiline(t *testing.T) { assert.Equal(t, int64(2), lineOffset) } +// HTML-like tags can collapse source blank lines during decoding. +func TestFragmentLineOffsetUsesOriginalData(t *testing.T) { + result := &detectors.Result{Raw: []byte("synthetic-secret-value-123456")} + result.SetPrimarySecretValue(`token = "synthetic-secret-value-123456"`) + chunk := &sources.Chunk{ + Data: []byte("# Date format:\n\ntoken = \"synthetic-secret-value-123456\""), + OriginalData: []byte("# Date format: \n\n\n\n\ntoken = \"synthetic-secret-value-123456\""), + } + + lineOffset, _ := FragmentLineOffset(chunk, result) + + assert.Equal(t, int64(5), lineOffset) +} + +// Some decoded secrets have no matching source byte sequence. +func TestFragmentLineOffsetFallsBackToDecodedData(t *testing.T) { + result := &detectors.Result{Raw: []byte("synthetic-secret-value-123456")} + chunk := &sources.Chunk{ + Data: []byte("decoded header\nsynthetic-secret-value-123456"), + OriginalData: []byte("encoded-source-data"), + } + + lineOffset, _ := FragmentLineOffset(chunk, result) + + assert.Equal(t, int64(1), lineOffset) +} + +// Decoding can change the newline count between duplicate matches. +func TestFragmentLineOffsetMapsOriginalDataOccurrence(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + chunk := &sources.Chunk{ + Data: []byte("synthetic-secret-value-123456\nsynthetic-secret-value-123456"), + OriginalData: []byte("synthetic-secret-value-123456\n\nsynthetic-secret-value-123456"), + } + results := []detectors.Result{{Raw: secret}, {Raw: secret}} + AssignDuplicateLineOffsets(chunk, results) + + lineOffset, _ := FragmentLineOffset(chunk, &results[1]) + + assert.Equal(t, int64(2), lineOffset) +} + +// The same value can occur in discarded markup and emitted text, so surrounding +// text has to identify which source occurrence the decoder kept. +func TestFragmentLineOffsetSkipsRemovedOriginalDataOccurrence(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + chunk := &sources.Chunk{ + Data: []byte("heading\nsynthetic-secret-value-123456"), + OriginalData: []byte("
\n" + + "

heading

\n

synthetic-secret-value-123456

"), + } + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.Equal(t, int64(2), lineOffset) +} + +// A dropped occurrence can sit after the surviving one, so a later source match is +// not automatically the right one. +func TestFragmentLineOffsetSkipsLaterRemovedOriginalDataOccurrence(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + chunk := &sources.Chunk{ + Data: []byte("heading\nsynthetic-secret-value-123456\ntrailer"), + OriginalData: []byte("

heading

\n

synthetic-secret-value-123456

\n" + + "
trailer
"), + } + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.Equal(t, int64(1), lineOffset) +} + +// UTF-8 decoding replaces line breaks alongside malformed bytes. +func TestFragmentLineOffsetUsesInvalidOriginalData(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + originalData := append([]byte("heading\n\xff\n"), secret...) + chunk := &sources.Chunk{ + Data: originalData, + OriginalData: originalData, + } + require.NotNil(t, (&decoders.UTF8{}).FromChunk(chunk)) + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.Equal(t, int64(2), lineOffset) +} + +// Mapping must count bytes across multibyte UTF-8 before the secret. +func TestFragmentLineOffsetUsesUTF8OriginalData(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + chunk := &sources.Chunk{ + Data: append([]byte("préface\n"), secret...), + OriginalData: append([]byte("préface\n\n"), secret...), + } + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.Equal(t, int64(2), lineOffset) +} + +// Replacement runes shift the decoded offset of the second match. +func TestFragmentLineOffsetMapsInvalidOriginalDataDuplicates(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + originalData := append([]byte("\xff\n"), secret...) + originalData = append(originalData, '\n', 0xfe, '\n') + originalData = append(originalData, secret...) + chunk := &sources.Chunk{Data: originalData, OriginalData: originalData} + require.NotNil(t, (&decoders.UTF8{}).FromChunk(chunk)) + results := []detectors.Result{{Raw: secret}, {Raw: secret}} + AssignDuplicateLineOffsets(chunk, results) + + lineOffset, _ := FragmentLineOffset(chunk, &results[1]) + + assert.Equal(t, int64(3), lineOffset) +} + +// Source markup can retain an ignore tag absent from decoded data. +func TestFragmentLineOffsetUsesOriginalDataIgnoreTag(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + chunk := &sources.Chunk{ + Data: []byte("synthetic-secret-value-123456\ntext"), + OriginalData: []byte("

synthetic-secret-value-123456

" + + "
text
"), + } + + _, ignored := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.True(t, ignored) +} + +func buildDroppedSpanChunk(t *rapid.T, secret []byte, decoys bool) (*sources.Chunk, int) { + spans := rapid.IntRange(1, 8).Draw(t, "spans") + target := rapid.IntRange(0, spans-1).Draw(t, "target") + + var original, decoded []byte + var secretOffset int + for i := range spans { + span := fmt.Appendf(nil, "

word%d\n

\n", i, i) + dropped := i != target && rapid.Bool().Draw(t, fmt.Sprintf("dropped%d", i)) + if i == target { + secretOffset = len(original) + len(span) + } + if i == target || (decoys && dropped && rapid.Bool().Draw(t, fmt.Sprintf("decoy%d", i))) { + span = append(append(span, secret...), '\n') + } + original = append(original, span...) + if !dropped { + decoded = append(decoded, span...) + } + } + return &sources.Chunk{Data: decoded, OriginalData: original}, secretOffset +} + +// Decoders drop whole spans, and the value they keep still has to land on its own +// source line however much text went missing around it. +func TestFragmentLineOffsetMapsDroppedSourceSpans(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + rapid.Check(t, func(t *rapid.T) { + chunk, secretOffset := buildDroppedSpanChunk(t, secret, false) + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.Equal(t, int64(bytes.Count(chunk.OriginalData[:secretOffset], []byte{'\n'})), lineOffset) + }) +} + +// When a dropped span holds the same value as a kept one the copies are only +// distinguishable by their surroundings, so the exact line is best-effort. What is +// not negotiable is that the reported line holds the value in the source. +func TestFragmentLineOffsetReportsLineHoldingSecret(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + rapid.Check(t, func(t *rapid.T) { + chunk, _ := buildDroppedSpanChunk(t, secret, true) + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + lines := bytes.Split(chunk.OriginalData, []byte{'\n'}) + require.Less(t, int(lineOffset), len(lines)) + assert.Contains(t, string(lines[lineOffset]), string(secret)) + }) +} + +// Generated binary prefixes exercise byte values without requiring valid UTF-8. +func TestFragmentLineOffsetMapsArbitraryOriginalData(t *testing.T) { + secret := []byte("synthetic-secret-value-123456") + rapid.Check(t, func(t *rapid.T) { + prefix := rapid.SliceOfN(rapid.Byte(), 0, 16).Draw(t, "prefix") + removed := rapid.SliceOfN(rapid.Byte(), 1, 16).Draw(t, "removed") + suffix := rapid.SliceOfN(rapid.Byte(), 0, 16).Draw(t, "suffix") + data := append(append(append(bytes.Clone(prefix), secret...), suffix...), '\n') + originalData := append(append(append(append(bytes.Clone(prefix), removed...), secret...), suffix...), '\n') + chunk := &sources.Chunk{Data: data, OriginalData: originalData} + + lineOffset, _ := FragmentLineOffset(chunk, &detectors.Result{Raw: secret}) + + assert.Equal(t, int64(bytes.Count(originalData[:len(prefix)+len(removed)], []byte{'\n'})), lineOffset) + }) +} + +// TestFragmentLineOffset_DuplicateSecrets verifies that when the same secret +// appears on multiple lines within a chunk, each result receives the correct +// line number rather than always reporting the first occurrence's line. +// Regression test for https://github.com/trufflesecurity/trufflehog/issues/2502 +func TestFragmentLineOffset_DuplicateSecrets(t *testing.T) { + secret := []byte("AKIA1234567890ABCDEF") + chunk := &sources.Chunk{ + Data: []byte("line1\n" + // line 0 + "line2\n" + // line 1 + "AKIA1234567890ABCDEF\n" + // line 2 (first occurrence) + "line4\n" + // line 3 + "AKIA1234567890ABCDEF\n" + // line 4 (second occurrence) + "line6\n" + // line 5 + "AKIA1234567890ABCDEF\n"), // line 6 (third occurrence) + } + + results := []detectors.Result{ + {Raw: secret}, + {Raw: secret}, + {Raw: secret}, + } + expectedLines := []int64{2, 4, 6} + + AssignDuplicateLineOffsets(chunk, results) + + seen := make(map[int64]bool) + for i, res := range results { + lineOffset, _ := FragmentLineOffset(chunk, &res) + assert.Equal(t, expectedLines[i], lineOffset, + "result[%d]: expected line %d but got %d", i, expectedLines[i], lineOffset) + assert.False(t, seen[lineOffset], + "result[%d]: line %d was already reported by a previous result (duplicate line number)", i, lineOffset) + seen[lineOffset] = true + } +} + +func TestAssignDuplicateLineOffsets(t *testing.T) { + chunk := &sources.Chunk{ + Data: []byte("aaa\nbbb\naaa\nccc\naaa\n"), + } + results := []detectors.Result{ + {Raw: []byte("aaa")}, + {Raw: []byte("aaa")}, + {Raw: []byte("aaa")}, + {Raw: []byte("bbb")}, + } + AssignDuplicateLineOffsets(chunk, results) + + // Duplicates get offsets assigned. + assert.True(t, results[0].HasChunkOffset()) + assert.Equal(t, int64(0), results[0].ChunkOffset()) + + assert.True(t, results[1].HasChunkOffset()) + assert.Equal(t, int64(8), results[1].ChunkOffset()) // "aaa\nbbb\n" = 8 bytes + + assert.True(t, results[2].HasChunkOffset()) + assert.Equal(t, int64(16), results[2].ChunkOffset()) // "aaa\nbbb\naaa\nccc\n" = 16 bytes + + // Unique secret does not get an offset. + assert.False(t, results[3].HasChunkOffset()) +} + +func TestFragmentLineOffset_DuplicateSecretsWithIgnoreTag(t *testing.T) { + secret := []byte("mysecret") + chunk := &sources.Chunk{ + Data: []byte("mysecret\nfoo\nmysecret trufflehog:ignore\nbar\nmysecret\n"), + } + results := []detectors.Result{ + {Raw: secret}, + {Raw: secret}, + {Raw: secret}, + } + AssignDuplicateLineOffsets(chunk, results) + + line0, ignored0 := FragmentLineOffset(chunk, &results[0]) + assert.Equal(t, int64(0), line0) + assert.False(t, ignored0) + + line1, ignored1 := FragmentLineOffset(chunk, &results[1]) + assert.Equal(t, int64(2), line1) + assert.True(t, ignored1) + + line2, ignored2 := FragmentLineOffset(chunk, &results[2]) + assert.Equal(t, int64(4), line2) + assert.False(t, ignored2) +} + func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) { data := make([]byte, 0, 4096) needle := []byte("needle") @@ -415,7 +705,7 @@ even more`, tmpFile, err := os.CreateTemp("", "test_aws_credentials") assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) + defer func() { _ = os.Remove(tmpFile.Name()) }() err = os.WriteFile(tmpFile.Name(), []byte(tt.content), os.ModeAppend) assert.NoError(t, err) @@ -463,10 +753,10 @@ func TestEngine_VersionedDetectorsVerifiedSecrets(t *testing.T) { tmpFile, err := os.CreateTemp("", "testfile") assert.Nil(t, err) - defer tmpFile.Close() - defer os.Remove(tmpFile.Name()) + defer func() { _ = tmpFile.Close() }() + defer func() { _ = os.Remove(tmpFile.Name()) }() - _, err = tmpFile.WriteString(fmt.Sprintf("test data using keyword %s", fakeDetectorKeyword)) + _, err = fmt.Fprintf(tmpFile, "test data using keyword %s", fakeDetectorKeyword) assert.NoError(t, err) const defaultOutputBufferSize = 64 @@ -507,8 +797,8 @@ func TestEngine_VersionedDetectorsVerifiedSecrets(t *testing.T) { func TestEngine_CustomDetectorsDetectorsVerifiedSecrets(t *testing.T) { tmpFile, err := os.CreateTemp("", "testfile") assert.Nil(t, err) - defer tmpFile.Close() - defer os.Remove(tmpFile.Name()) + defer func() { _ = tmpFile.Close() }() + defer func() { _ = os.Remove(tmpFile.Name()) }() _, err = tmpFile.WriteString("test stuff") assert.Nil(t, err) @@ -635,6 +925,39 @@ func TestProcessResult_IgnoreLinePresent_NothingGenerated(t *testing.T) { assert.Empty(t, e.results) } +func TestProcessResult_IgnoreLinePresentWithNoIgnoreTag_ResultGenerated(t *testing.T) { + // Arrange: Create an engine that does not honor ignore tags + e := Engine{results: make(chan detectors.ResultWithMetadata, 1), noIgnoreTag: true} + + // Arrange: Create a Chunk + chunk := sources.Chunk{ + Data: []byte("swordfish trufflehog:ignore"), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{ + Line: 1, + }, + }, + }, + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + } + + // Arrange: Create a Result + result := detectors.Result{ + Raw: []byte("swordfish"), + Verified: true, + } + + // Act + e.processResult(context.AddLogger(t.Context()), result, chunk, 0, "", nil) + + // Assert that the result was reported anyway, with its line number still set + require.Len(t, e.results, 1) + r := <-e.results + assert.Equal(t, []byte("swordfish"), r.Raw) + assert.Equal(t, int64(1), r.SourceMetadata.GetGit().GetLine()) +} + func TestProcessResult_AllFieldsCopied(t *testing.T) { // Arrange: Create an engine e := Engine{results: make(chan detectors.ResultWithMetadata, 1)} @@ -831,15 +1154,6 @@ func TestEngine_FalsePositivesRetainedCorrectly(t *testing.T) { retainFalsePositives: false, wantUnverifiedSecretCount: 0, }, - { - name: "overlap, retain false positives", - detectors: []detectors.Detector{ - passthroughDetector{detectorType: detector_typepb.DetectorType(-1), keywords: []string{"sample"}}, - passthroughDetector{detectorType: detector_typepb.DetectorType(-2), keywords: []string{"ample"}}, - }, - retainFalsePositives: true, - wantUnverifiedSecretCount: 2, - }, { name: "overlap, do not retain false positives", detectors: []detectors.Detector{ @@ -1181,7 +1495,10 @@ func (c customCleaner) Type() detector_typepb.DetectorType { return detector_typ func (customCleaner) Description() string { return "" } -func (c customCleaner) CleanResults([]detectors.Result) []detectors.Result { +func (c customCleaner) CleanResults(result []detectors.Result, verficationEnabled bool) []detectors.Result { + if !verficationEnabled { + return []detectors.Result{{}} + } return []detectors.Result{} } func (c customCleaner) ShouldCleanResultsIrrespectiveOfConfiguration() bool { return c.ignoreConfig } @@ -1191,6 +1508,7 @@ func TestFilterResults_CustomCleaner(t *testing.T) { name string cleaningConfigured bool ignoreConfig bool + verify bool resultsToClean []detectors.Result wantResults []detectors.Result }{ @@ -1198,6 +1516,7 @@ func TestFilterResults_CustomCleaner(t *testing.T) { name: "respect config to clean", cleaningConfigured: true, ignoreConfig: false, + verify: true, resultsToClean: []detectors.Result{{}}, wantResults: []detectors.Result{}, }, @@ -1205,6 +1524,7 @@ func TestFilterResults_CustomCleaner(t *testing.T) { name: "respect config to not clean", cleaningConfigured: false, ignoreConfig: false, + verify: true, resultsToClean: []detectors.Result{{}}, wantResults: []detectors.Result{{}}, }, @@ -1212,9 +1532,18 @@ func TestFilterResults_CustomCleaner(t *testing.T) { name: "clean irrespective of config", cleaningConfigured: false, ignoreConfig: true, + verify: true, resultsToClean: []detectors.Result{{}}, wantResults: []detectors.Result{}, }, + { + name: "clean irrespective of config with verification disabled", + cleaningConfigured: false, + ignoreConfig: true, + verify: false, + resultsToClean: []detectors.Result{{}}, + wantResults: []detectors.Result{{}}, + }, } for _, tt := range testCases { @@ -1227,9 +1556,10 @@ func TestFilterResults_CustomCleaner(t *testing.T) { engine := Engine{ filterUnverified: tt.cleaningConfigured, retainFalsePositives: true, + verify: tt.verify, } - cleaned := engine.filterResults(context.Background(), &match, tt.resultsToClean) + cleaned := engine.filterResults(context.Background(), "detect", &match, tt.resultsToClean) assert.ElementsMatch(t, tt.wantResults, cleaned) }) @@ -1381,6 +1711,9 @@ func TestEngineInitializesCloudProviderDetectors(t *testing.T) { detector_typepb.DetectorType_HashiCorpVaultAuth: {}, detector_typepb.DetectorType_JiraDataCenterPAT: {}, detector_typepb.DetectorType_ConfluenceDataCenter: {}, + detector_typepb.DetectorType_BitbucketDataCenter: {}, + detector_typepb.DetectorType_HashiCorpVaultBatchToken: {}, + detector_typepb.DetectorType_HashiCorpVaultToken: {}, // these do not have any cloud endpoint } @@ -1420,6 +1753,18 @@ def test_something(): assert connection_string == "postgres://master_user:master_password@hostname:1234/main" # trufflehog:ignore`, expectedFindings: 0, }, + { + name: "ignore postgres url without explicit port", + content: ` +# tests/example_false_positive.py + +def test_something(): + connection_string = "who-cares" + + # The detector normalizes this URL to include :5432, but the ignore tag should still be honored. + assert connection_string == "postgres://master_user:master_password@hostname/main" # trufflehog:ignore`, + expectedFindings: 0, + }, { name: "ignore not on secret line", content: ` @@ -1444,7 +1789,7 @@ def test_something(): tmpFile, err := os.CreateTemp("", "test_creds") assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) + defer func() { _ = os.Remove(tmpFile.Name()) }() err = os.WriteFile(tmpFile.Name(), []byte(tt.content), os.ModeAppend) assert.NoError(t, err) @@ -1557,7 +1902,7 @@ func TestEngine_DetectChunk_UsesVerifyFlag(t *testing.T) { // Assert: Confirm that a result was generated and that it has the expected verify flag. select { case result := <-e.results: - assert.Equal(t, tc.verify, result.Result.Verified) + assert.Equal(t, tc.verify, result.Verified) default: t.Errorf("expected a result but did not get one") } @@ -1679,7 +2024,7 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t // Assert: Confirm that every generated result is unverified (because overlap detection precluded it). for result := range e.results { - assert.False(t, result.Result.Verified) + assert.False(t, result.Verified) } // Assert: Confirm that every generated detectable chunk's Chunk.SourceVerify flag is unchanged and that its @@ -1899,3 +2244,230 @@ func TestEngine_IterativeDecoding(t *testing.T) { }) } } + +// captureDispatcher records every dispatched result for assertion in tests. It +// is safe for concurrent use because the engine runs many notifier workers +// against a single dispatcher. +type captureDispatcher struct { + mu sync.Mutex + results []detectors.ResultWithMetadata +} + +func (d *captureDispatcher) Dispatch(_ context.Context, result detectors.ResultWithMetadata) error { + d.mu.Lock() + defer d.mu.Unlock() + d.results = append(d.results, result) + return nil +} + +// TestNotifierWorker_ReverifiedResultsBypassDedupe verifies that the notifier's +// dedupe cache is skipped when a result carries a non-zero SecretID — i.e., when +// it originated from reverification — so that the dispatcher sees every +// reverification result even when the underlying secret has not changed. +func TestNotifierWorker_ReverifiedResultsBypassDedupe(t *testing.T) { + tests := []struct { + name string + secretID int64 + wantDispatch int + }{ + { + name: "non-reverified duplicates are deduplicated", + secretID: 0, + wantDispatch: 1, + }, + { + name: "reverified duplicates bypass the dedupe cache", + secretID: 42, + wantDispatch: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache, err := lru.New[string, struct{}](16) + require.NoError(t, err) + + disp := &captureDispatcher{} + e := &Engine{ + results: make(chan detectors.ResultWithMetadata, 4), + dedupeCache: cache, + dispatcher: disp, + notifyVerifiedResults: true, + notifyUnverifiedResults: true, + notifyUnknownResults: true, + } + + result := detectors.ResultWithMetadata{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{Line: 1}, + }, + }, + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + SecretID: tt.secretID, + Result: detectors.Result{ + DetectorType: detector_typepb.DetectorType(-1), + Raw: []byte("a-secret"), + Verified: true, + }, + } + + // Push the same result twice — identical hash inputs. + e.results <- result + e.results <- result + close(e.results) + + e.notifierWorker(context.Background()) + + assert.Equal(t, tt.wantDispatch, len(disp.results)) + }) + } +} + +// TestNotifierWorker_ConcurrentDuplicatesDispatchedOnce verifies that when many +// notifier workers share the dedupe cache, identical results are dispatched +// exactly once. The check-and-insert must be atomic; a separate lookup and add +// lets two workers both miss and both dispatch. +// +// The race exists only on a key's first sighting, so the test uses many distinct +// secrets and queues each one's copies back to back, giving every key its own +// chance for workers to collide. It guards a logical race rather than a data +// race, so -race does not flag the non-atomic version. +func TestNotifierWorker_ConcurrentDuplicatesDispatchedOnce(t *testing.T) { + const ( + numSecrets = 1000 + numCopies = 16 + numWorkers = 16 + ) + + // Sized to hold every key so eviction cannot cause a re-dispatch. + cache, err := lru.New[string, struct{}](numSecrets) + require.NoError(t, err) + + disp := &captureDispatcher{} + e := &Engine{ + results: make(chan detectors.ResultWithMetadata, numSecrets*numCopies), + dedupeCache: cache, + dispatcher: disp, + notifyVerifiedResults: true, + notifyUnverifiedResults: true, + notifyUnknownResults: true, + } + + // Fill and close the channel before starting workers so they all contend + // on the cache at once instead of idling on an empty channel. SecretID + // stays 0 so every copy goes through the dedupe cache. + for i := range numSecrets { + result := detectors.ResultWithMetadata{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{Line: 1}, + }, + }, + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + Result: detectors.Result{ + DetectorType: detector_typepb.DetectorType(-1), + Raw: []byte(fmt.Sprintf("secret-%d", i)), + Verified: true, + }, + } + for range numCopies { + e.results <- result + } + } + close(e.results) + + var wg sync.WaitGroup + for range numWorkers { + wg.Add(1) + go func() { + defer wg.Done() + e.notifierWorker(context.Background()) + }() + } + wg.Wait() + + assert.Equal(t, numSecrets, len(disp.results)) +} + +func setupSourceMappingBench(size int, decode bool) (*sources.Chunk, *detectors.Result) { + secret := []byte("synthetic-secret-value-123456") + var original, decoded []byte + for i := 0; len(original) < size; i++ { + original = append(original, fmt.Sprintf("

line%d

\n", i)...) + decoded = append(decoded, fmt.Sprintf("line%d\n", i)...) + } + original = append(original, "

"...) + original = append(original, secret...) + original = append(original, "

\n"...) + decoded = append(decoded, secret...) + decoded = append(decoded, '\n') + if !decode { + decoded = original + } + return &sources.Chunk{Data: decoded, OriginalData: original}, &detectors.Result{Raw: secret} +} + +func benchmarkSourceMapping(b *testing.B, size int, decode bool) { + chunk, result := setupSourceMappingBench(size, decode) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = FragmentLineOffset(chunk, result) + } +} + +func BenchmarkFragmentLineOffset_Identical_Small(b *testing.B) { + benchmarkSourceMapping(b, 64, false) +} + +func BenchmarkFragmentLineOffset_Identical_DefaultChunkSize(b *testing.B) { + benchmarkSourceMapping(b, sources.DefaultChunkSize, false) +} + +func BenchmarkFragmentLineOffset_Identical_Large(b *testing.B) { + benchmarkSourceMapping(b, 64*1024, false) +} + +func BenchmarkFragmentLineOffset_Diffed_Small(b *testing.B) { + benchmarkSourceMapping(b, 64, true) +} + +func BenchmarkFragmentLineOffset_Diffed_DefaultChunkSize(b *testing.B) { + benchmarkSourceMapping(b, sources.DefaultChunkSize, true) +} + +func BenchmarkFragmentLineOffset_Diffed_MaxRealisticChunk(b *testing.B) { + benchmarkSourceMapping(b, sources.TotalChunkSize, true) +} + +func BenchmarkFragmentLineOffset_Diffed_Large(b *testing.B) { + benchmarkSourceMapping(b, 64*1024, true) +} + +func setupAmbiguousSourceBench(size int) (*sources.Chunk, *detectors.Result) { + secret := []byte("synthetic-secret-value-123456") + original := []byte("
\n") + var decoded []byte + for i := 0; len(original) < size; i++ { + original = append(original, fmt.Sprintf("

line%d

\n", i)...) + decoded = append(decoded, fmt.Sprintf("line%d\n", i)...) + } + original = append(original, "

"...) + original = append(original, secret...) + original = append(original, "

\n"...) + decoded = append(decoded, secret...) + decoded = append(decoded, '\n') + return &sources.Chunk{Data: decoded, OriginalData: original}, &detectors.Result{Raw: secret} +} + +// The same value occurs in both discarded markup and emitted text, so the source +// occurrence has to be picked by its surroundings. +func BenchmarkFragmentLineOffset_Ambiguous_DefaultChunkSize(b *testing.B) { + chunk, result := setupAmbiguousSourceBench(sources.DefaultChunkSize) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = FragmentLineOffset(chunk, result) + } +} diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go index e1b6ce51d9da..cdbc96de0a99 100644 --- a/pkg/engine/git_test.go +++ b/pkg/engine/git_test.go @@ -41,7 +41,7 @@ func TestGitEngine(t *testing.T) { if err != nil { t.Error(err) } - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -130,7 +130,7 @@ func TestGitEngineWithMirrorAndBareClones(t *testing.T) { if err != nil { t.Fail() } - defer os.RemoveAll(parent) + defer func() { _ = os.RemoveAll(parent) }() localRepo := filepath.Join(parent, "test_keys.git") cloneCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() diff --git a/pkg/engine/github.go b/pkg/engine/github.go index 30b090913e1b..9ccc966bacab 100644 --- a/pkg/engine/github.go +++ b/pkg/engine/github.go @@ -32,6 +32,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) (source ClonePath: c.ClonePath, NoCleanup: c.NoCleanup, IgnoreGists: c.IgnoreGists, + ExcludeArchived: c.ExcludeArchived, PrintLegacyJson: c.PrintLegacyJSON, } diff --git a/pkg/engine/huggingface.go b/pkg/engine/huggingface.go index 9a20fe27f713..e735be54f145 100644 --- a/pkg/engine/huggingface.go +++ b/pkg/engine/huggingface.go @@ -10,7 +10,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources/huggingface" ) -// HuggingFaceConfig represents the configuration for HuggingFace. +// HuggingfaceConfig represents the configuration for HuggingFace. type HuggingfaceConfig struct { Endpoint string Models []string @@ -24,16 +24,20 @@ type HuggingfaceConfig struct { IgnoreSpaces []string IncludeDatasets []string IgnoreDatasets []string + Buckets []string + IncludeBuckets []string + IgnoreBuckets []string SkipAllModels bool SkipAllSpaces bool SkipAllDatasets bool + SkipAllBuckets bool IncludeDiscussions bool IncludePrs bool Token string Concurrency int } -// ScanGitHub scans HuggingFace with the provided options. +// ScanHuggingface scans HuggingFace with the provided options. func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) (sources.JobProgressRef, error) { connection := sourcespb.Huggingface{ Endpoint: c.Endpoint, @@ -48,9 +52,13 @@ func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) (sour IgnoreSpaces: c.IgnoreSpaces, IncludeDatasets: c.IncludeDatasets, IgnoreDatasets: c.IgnoreDatasets, + Buckets: c.Buckets, + IncludeBuckets: c.IncludeBuckets, + IgnoreBuckets: c.IgnoreBuckets, SkipAllModels: c.SkipAllModels, SkipAllSpaces: c.SkipAllSpaces, SkipAllDatasets: c.SkipAllDatasets, + SkipAllBuckets: c.SkipAllBuckets, IncludeDiscussions: c.IncludeDiscussions, IncludePrs: c.IncludePrs, } diff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.go index f2f43b3ccce9..b929c4f01b8a 100644 --- a/pkg/engine/metrics.go +++ b/pkg/engine/metrics.go @@ -16,7 +16,7 @@ var ( Name: "detector_execution_count", Help: "Total number of times a detector has been executed.", }, - []string{"detector_name", "job_id", "source_name"}, + []string{"detector_name", "source_type", "source_name"}, ) // Note this is the time taken to call FromData on each detector, not necessarily the time taken @@ -103,4 +103,51 @@ var ( Help: "Time taken to notify a chunk in milliseconds.", Buckets: prometheus.ExponentialBuckets(5, 2, 12), }) + + // Pipeline flow metrics tracks chunks/results entering each stage, + // where they are dropped, and where they exit. + chunksEnteredStage = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "chunks_entered_stage_total", + Help: "Total number of chunks (or per-detector work items) entering each pipeline stage.", + }, + []string{"stage", "source_type"}, + ) + + chunksDropped = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "chunks_dropped_total", + Help: "Total number of chunks dropped by pipeline stage and reason.", + }, + []string{"stage", "reason", "source_type"}, + ) + + resultsProduced = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "results_produced_total", + Help: "Total number of raw detector results produced by the detect stage (before filtering).", + }, + []string{"detector_name"}, + ) + + resultsDropped = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "results_dropped_total", + Help: "Total number of detector results dropped by pipeline stage and reason.", + }, + []string{"stage", "reason", "detector_name"}, + ) + + resultsDispatched = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "result_dispatch_attempts_count", + Help: "Count of result dispatch attempts.", + }, + []string{"detector_name", "verified", "success"}, + ) ) diff --git a/pkg/engine/runtime_collector.go b/pkg/engine/runtime_collector.go new file mode 100644 index 000000000000..8bc40f0c3f5e --- /dev/null +++ b/pkg/engine/runtime_collector.go @@ -0,0 +1,201 @@ +package engine + +import ( + "errors" + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" +) + +// Channel and worker pool label values exposed by runtimeCollector. These +// names appear in metric labels, so external dashboards depend on them being +// stable. +const ( + channelSourceChunks = "source_chunks_chan" + channelDetectableChunks = "detectable_chunks_chan" + channelVerificationOverlapChunks = "verification_overlap_chunks_chan" + channelResults = "results_chan" + + workerEngine = "engine_workers" + workerDetector = "detector_workers" + workerVerificationOverlap = "verification_overlap_workers" + workerNotifier = "notifier_workers" + workerSources = "source_workers" +) + +// runtimeCollector exposes live engine internals (channel queue depths, +// channel capacities, configured worker pool sizes, and aggregate scan +// counters) to Prometheus. Values are sampled lazily on each scrape via +// len/cap and atomic loads, so there is no background goroutine overhead +// between scrapes. +type runtimeCollector struct { + engine *Engine + + channelSize *prometheus.Desc + channelCapacity *prometheus.Desc + workerCount *prometheus.Desc + activeSources *prometheus.Desc + + bytesScanned *prometheus.Desc + chunksScanned *prometheus.Desc + verifiedSecretsFound *prometheus.Desc + unverifiedSecretsFound *prometheus.Desc +} + +// newRuntimeCollector constructs a runtimeCollector bound to the given +// Engine. The metric descriptors are built once here; each subsequent Collect +// call reuses them and only samples fresh values. +func newRuntimeCollector(e *Engine) *runtimeCollector { + fq := func(name string) string { + return prometheus.BuildFQName(common.MetricsNamespace, common.MetricsSubsystem, name) + } + return &runtimeCollector{ + engine: e, + channelSize: prometheus.NewDesc( + fq("engine_channel_size"), + "Current number of items buffered in an engine channel.", + []string{"channel"}, nil, + ), + channelCapacity: prometheus.NewDesc( + fq("engine_channel_capacity"), + "Buffer capacity of an engine channel.", + []string{"channel"}, nil, + ), + workerCount: prometheus.NewDesc( + fq("engine_worker_count"), + "Configured number of workers in an engine worker pool.", + []string{"worker_type"}, nil, + ), + activeSources: prometheus.NewDesc( + fq("engine_active_sources"), + "Number of sources currently running.", + nil, nil, + ), + bytesScanned: prometheus.NewDesc( + fq("engine_bytes_scanned"), + "Total bytes scanned by the engine since Start.", + nil, nil, + ), + chunksScanned: prometheus.NewDesc( + fq("engine_chunks_scanned"), + "Total chunks scanned by the engine since Start.", + nil, nil, + ), + verifiedSecretsFound: prometheus.NewDesc( + fq("engine_verified_secrets_found"), + "Total verified secrets found by the engine since Start.", + nil, nil, + ), + unverifiedSecretsFound: prometheus.NewDesc( + fq("engine_unverified_secrets_found"), + "Total unverified secrets found by the engine since Start.", + nil, nil, + ), + } +} + +// Describe implements prometheus.Collector by emitting every metric +// descriptor this collector will ever produce. The registry uses these +// descriptors to detect conflicts at registration time. +func (c *runtimeCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.channelSize + ch <- c.channelCapacity + ch <- c.workerCount + ch <- c.activeSources + ch <- c.bytesScanned + ch <- c.chunksScanned + ch <- c.verifiedSecretsFound + ch <- c.unverifiedSecretsFound +} + +// Collect implements prometheus.Collector and is invoked by the registry on +// every scrape. It samples the engine's channels via len/cap, reads scan +// counters via atomic.Load, and emits one metric per (descriptor, label set) +// combination. +func (c *runtimeCollector) Collect(ch chan<- prometheus.Metric) { + e := c.engine + + sourceChunks := e.sourceManager.Chunks() + channels := []struct { + name string + size int + capacity int + }{ + {channelSourceChunks, len(sourceChunks), cap(sourceChunks)}, + {channelDetectableChunks, len(e.detectableChunksChan), cap(e.detectableChunksChan)}, + {channelVerificationOverlapChunks, len(e.verificationOverlapChunksChan), cap(e.verificationOverlapChunksChan)}, + {channelResults, len(e.results), cap(e.results)}, + } + for _, q := range channels { + ch <- prometheus.MustNewConstMetric(c.channelSize, prometheus.GaugeValue, float64(q.size), q.name) + ch <- prometheus.MustNewConstMetric(c.channelCapacity, prometheus.GaugeValue, float64(q.capacity), q.name) + } + + workers := []struct { + name string + count int + }{ + {workerEngine, e.concurrency}, + {workerDetector, e.concurrency * e.detectorWorkerMultiplier}, + {workerVerificationOverlap, e.concurrency * e.verificationOverlapWorkerMultiplier}, + {workerNotifier, e.concurrency * e.notificationWorkerMultiplier}, + // SourceManager treats its concurrency limit as a semaphore rather than a + // fixed worker pool, but reporting it here lets dashboards ratio it against + // activeSources to spot source-side saturation. + {workerSources, e.sourceManager.MaxConcurrentSources()}, + } + for _, w := range workers { + ch <- prometheus.MustNewConstMetric(c.workerCount, prometheus.GaugeValue, float64(w.count), w.name) + } + + ch <- prometheus.MustNewConstMetric( + c.activeSources, prometheus.GaugeValue, + float64(e.sourceManager.ConcurrentSources()), + ) + + // Scan counters are written via atomic.AddUint64 on runtimeMetrics; read + // them with atomic.LoadUint64 to stay cheap on scrape and avoid the RW mutex. + m := &e.metrics.Metrics + ch <- prometheus.MustNewConstMetric(c.bytesScanned, prometheus.CounterValue, float64(atomic.LoadUint64(&m.BytesScanned))) + ch <- prometheus.MustNewConstMetric(c.chunksScanned, prometheus.CounterValue, float64(atomic.LoadUint64(&m.ChunksScanned))) + ch <- prometheus.MustNewConstMetric(c.verifiedSecretsFound, prometheus.CounterValue, float64(atomic.LoadUint64(&m.VerifiedSecretsFound))) + ch <- prometheus.MustNewConstMetric(c.unverifiedSecretsFound, prometheus.CounterValue, float64(atomic.LoadUint64(&m.UnverifiedSecretsFound))) +} + +// registerRuntimeMetrics installs the engine's runtime collector into the +// default Prometheus registry. If a collector with identical descriptors is +// already registered (e.g. a previous engine in the same process that didn't +// call Finish), the stale collector is evicted and replaced. Without eviction +// the stale collector would pin a dead engine in memory and permanently block +// future engines from exposing metrics. +func (e *Engine) registerRuntimeMetrics(ctx context.Context) { + collector := newRuntimeCollector(e) + err := prometheus.DefaultRegisterer.Register(collector) + if err != nil { + var already prometheus.AlreadyRegisteredError + if !errors.As(err, &already) { + ctx.Logger().Error(err, "failed to register engine runtime metrics") + return + } + ctx.Logger().V(2).Info("evicting stale engine runtime metrics collector") + prometheus.DefaultRegisterer.Unregister(already.ExistingCollector) + if err := prometheus.DefaultRegisterer.Register(collector); err != nil { + ctx.Logger().Error(err, "failed to register engine runtime metrics after eviction") + return + } + } + e.runtimeCollector = collector +} + +// unregisterRuntimeMetrics removes the engine's runtime collector from the +// default Prometheus registry. Safe to call when registration was skipped. +func (e *Engine) unregisterRuntimeMetrics() { + if e.runtimeCollector == nil { + return + } + prometheus.DefaultRegisterer.Unregister(e.runtimeCollector) + e.runtimeCollector = nil +} diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go index 1f6fe5b6dada..fdee09da8eb5 100644 --- a/pkg/engine/s3.go +++ b/pkg/engine/s3.go @@ -54,6 +54,14 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) (sources.JobPro connection.Roles = c.Roles } + connection.IncludePrefixes = c.IncludePrefixes + connection.ExcludePrefixes = c.ExcludePrefixes + connection.IncludeExtensions = c.IncludeExtensions + connection.ExcludeExtensions = c.ExcludeExtensions + + connection.Endpoint = c.Endpoint + connection.Region = c.Region + var conn anypb.Any err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) if err != nil { diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index ce392112f846..620f83e8b9df 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -6,18 +6,57 @@ import ( ) var ( - ForceSkipBinaries atomic.Bool - ForceSkipArchives atomic.Bool - GitCloneTimeoutDuration atomic.Int64 - SkipAdditionalRefs atomic.Bool - EnableAPKHandler atomic.Bool - UserAgentSuffix AtomicString - UseSimplifiedGitlabEnumeration atomic.Bool - UseGitMirror atomic.Bool - GitlabProjectsPerPage atomic.Int64 - UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments - HTMLDecoderEnabled atomic.Bool - CustomHeaders AtomicHeader + ForceSkipBinaries atomic.Bool + ForceSkipArchives atomic.Bool + GitCloneTimeoutDuration atomic.Int64 + SkipAdditionalRefs atomic.Bool + EnableAPKHandler atomic.Bool + UserAgentSuffix AtomicString + UseSimplifiedGitlabEnumeration atomic.Bool + UseGitMirror atomic.Bool + UseGitLowMemoryScan atomic.Bool + GitlabProjectsPerPage atomic.Int64 + UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments + HTMLDecoderEnabled atomic.Bool + PineconeDetectorEnabled atomic.Bool + CloudinaryDetectorEnabled atomic.Bool + GitLabOAuthDetectorEnabled atomic.Bool + SonarCloudV2DetectorEnabled atomic.Bool + EnigmaDetectorEnabled atomic.Bool + DatadogApiKeyDetectorEnabled atomic.Bool + TlyDetectorEnabled atomic.Bool + WitDetectorEnabled atomic.Bool + RevDetectorEnabled atomic.Bool + UserDetectorEnabled atomic.Bool + BraintrustDetectorEnabled atomic.Bool + PgAnalyzeReadKeyDetectorEnabled atomic.Bool + RedHatPyxisDetectorEnabled atomic.Bool + OctopusDeployDetectorEnabled atomic.Bool + DropUnverifiedJWTResults atomic.Bool + OpenRouterDetectorEnabled atomic.Bool + NewRelicInsightsInsertKeyDetectorEnabled atomic.Bool + DuffelTokenDetectorEnabled atomic.Bool + ShippoDetectorEnabled atomic.Bool + IPInfoDetectorEnabled atomic.Bool + LobDetectorEnabled atomic.Bool + HashiCorpVaultBatchTokenDetectorEnabled atomic.Bool + HashiCorpVaultTokenDetectorEnabled atomic.Bool + CloudflareApiTokenV2DetectorEnabled atomic.Bool + CloudflareGlobalApiKeyV2DetectorEnabled atomic.Bool + DuoDetectorEnabled atomic.Bool + FigmaV3DetectorEnabled atomic.Bool + NewRelicLicenseKeyDetectorEnabled atomic.Bool + NewRelicBrowserKeyDetectorEnabled atomic.Bool + NewRelicUserKeyDetectorEnabled atomic.Bool + NewRelicInsightsQueryKeyDetectorEnabled atomic.Bool + NewRelicMobileAppTokenDetectorEnabled atomic.Bool + MSTeamsWebhookV2DetectorEnabled atomic.Bool + SolarwindsDetectorEnabled atomic.Bool + ResendDetectorEnabled atomic.Bool + WeightsAndBiasesV2DetectorEnabled atomic.Bool + HumioAPITokenDetectorEnabled atomic.Bool + EnableS3UnitProgress atomic.Bool // count each S3 bucket during a unit scan to report progress inside it + CustomHeaders AtomicHeader ) type AtomicString struct { diff --git a/pkg/gitparse/README.md b/pkg/gitparse/README.md new file mode 100644 index 000000000000..d59575c4e348 --- /dev/null +++ b/pkg/gitparse/README.md @@ -0,0 +1,123 @@ +# gitparse + +`gitparse` turns a git repository's history into a stream of `*Diff` values, one for each +changed file in each commit. It is the front door for the git based sources: everything +TruffleHog scans out of a repository's history comes through this package. + +It works by running the `git` binary and reading its text output, rather than reading +git's object files directly. Parsing text is not pretty, but `git log --patch` already +handles renames, binary files, path filters and merges, and rewriting all of that +ourselves would be a much bigger thing to get wrong. + +## What comes out + +A `Diff` is one file's added content inside one commit, along with the details of that +commit: + +```go +diffChan, err := parser.RepoPath(ctx, repoPath, "", "", nil, false) +for diff := range diffChan { + diff.Commit.Hash // the commit this change belongs to + diff.PathB // the path of the changed file + diff.LineStart // where in the file the hunk starts + diff.ReadCloser() // the added lines, and nothing else +} +``` + +Only added lines are kept. Removed and unchanged lines are thrown away, because a secret +that was deleted was already there in the commit that added it, and that commit is in the +stream too. + +Commit details are attached to every diff. A commit with no file changes at all, such as +a merge or an empty commit, is still sent once on its own, because its message, author and +notes can hold a secret even when no file changed. + +Diff content goes to one of two writers, chosen by the caller. `buffer_writer` keeps it in +memory and is the default. `buffered_file_writer` spills to disk past a size and is turned +on with `UseCustomContentWriter`. Repositories with big files want the second one. + +## How the log is produced + +There are two ways, and the caller picks. + +**The normal way** is one `git log --patch` for the whole repository. Simple, fast, and +fine for most repositories. + +**The lower memory way** is turned on with `UseLowMemoryScan`, which the CLI exposes as +`--git-low-memory-scan`. It exists because one `git log --patch` keeps a little state for +every commit it walks and never gives any of it back, so on a repository with millions of +commits it grows until the machine kills it. + +That mode splits the work in two: + +1. **List the commits.** `git rev-list` prints the hashes we want and nothing else. The + hashes come back in groups, so the next step can start before the whole history has + been walked. + +2. **Make the patches.** Each group of hashes is handed to its own short lived `git log` + process, which reads them on standard input. Each process only holds state for its own + group and then exits, so this step costs the same no matter how long the history is. + +The results of all those processes are joined back into the single channel the caller +sees, so nothing above this package needs to know which way was used. + +## Things that look like details but are not + +Each of these came from a test or a measurement that said otherwise. + +**Listing uses `git rev-list`, not `git log`.** They can both print hashes, but `git log` +sets up git's diff machinery as soon as a diff option is present, and then compares files +in every commit just to decide which commits to print. `rev-list` never does that. This is +the step that uses the most memory on long histories, so the difference matters, and it is +large. + +**Diff options stay out of the listing step.** Options like `--diff-filter=AM` live with +the patch options, not the commit picking options. `rev-list` rejects them anyway. Leaving +them out means the listing picks up a few extra commits whose changes are all filtered +away, and the `git log` that makes the patches drops those commits itself, exactly as the +single command form does. The end result is the same commits, worked out in the cheaper +place. + +**Patches come from `git log --no-walk=unsorted`, not `git show`.** They print the same +thing for ordinary commits, but only `git log` applies `--diff-filter` to whole commits, +so using `git show` here would scan more commits than the normal way does. `unsorted` +matters too: plain `--no-walk` re-sorts each group by commit date, which scrambles the +order across groups. + +**Hashes go in on standard input, not as arguments.** A command line has a length limit, +and on Windows it is short enough to cap a group at a few dozen commits. Standard input +has no such limit, so groups can be thousands of commits and full hashes can be used +instead of shortened ones. Fewer, bigger groups mean fewer git processes to start. + +**Path filters go to both steps.** The listing needs them to pick the same commits the +single command would have shown, and the patch step needs them to decide which files show +up. + +**The last commit of a stream needs finishing off.** The parser normally completes a +commit when it sees the next one begin. The last commit never gets that, so it is +completed at the end of the stream instead. Without this, a commit with no file changes +was dropped whenever it landed at the end, which used to be rare with one long stream and +became common once the log was split into groups. + +## Stopping early + +A scan can stop before the diffs run out, when it hits its maximum depth or reaches its +base commit. A Go channel cannot tell you that its reader has gone away, so the scan +cancels its context on the way out, and that is what shuts down the git processes still +producing diffs. + +Cancelling also keeps the listing step short. It only ever runs a few groups ahead of what +is being consumed, because the channels between the steps are unbuffered, so a scan that +stops early never walks the whole history. + +## Tests + +```sh +go test ./pkg/gitparse/ +``` + +The tests in `lowmemory_test.go` hold the lower memory mode to one rule: it must produce +exactly what the single command form produces, with the same commits, paths and content, +wherever the group boundaries land. They run with tiny group sizes on purpose, since a +group size of one puts every commit at the end of a stream at once, which is where the +awkward cases live. diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go index 3c6a0c804500..24e5f6611cf8 100644 --- a/pkg/gitparse/gitparse.go +++ b/pkg/gitparse/gitparse.go @@ -3,11 +3,13 @@ package gitparse import ( "bufio" "bytes" + "errors" "fmt" "io" "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "time" @@ -33,6 +35,14 @@ const ( // defaultWaitDelay is the default time to wait after context cancellation before forcefully killing git processes. defaultWaitDelay = 5 * time.Second + + // logGroupSize is the number of commits per `git log` in the lower-memory scan mode. + // + // The hashes are fed in on stdin rather than as arguments, so a group is not + // limited by how long a command line may be and we can use full hashes. Bigger + // groups mean fewer git processes to start; each one still only holds state for + // its own group, which is what keeps memory flat however long the history is. + logGroupSize = 5000 ) // contentWriter defines a common interface for writing, reading, and managing diff content. @@ -130,6 +140,12 @@ type Parser struct { waitDelay time.Duration useCustomContentWriter bool + lowMemoryScan bool + + // groupSize is how many commits go to each `git log` in the lower-memory scan. + // Zero means logGroupSize. Only the tests set it, so they can put a group + // boundary wherever they need one. + groupSize int } type ParseState int @@ -191,6 +207,13 @@ func UseCustomContentWriter() Option { return func(parser *Parser) { parser.useCustomContentWriter = true } } +// UseLowMemoryScan sets the scan to optimize for limited memory at the cost of speed (currently up to 9%) +func UseLowMemoryScan() Option { + return func(parser *Parser) { + parser.lowMemoryScan = true + } +} + // WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will // be truncated. func WithMaxDiffSize(maxDiffSize int64) Option { @@ -233,60 +256,311 @@ func NewParser(options ...Option) *Parser { return parser } +type gitArgs struct { + env []string + global []string + log []string + show []string + paths []string +} + // RepoPath parses the output of the `git log` command for the `source` path. -// The Diff chan will return diffs in the order they are parsed from the log. +// The Diff chan will return diffs in the order they are parsed from the log, +// though the diffs are generated using `git show` in groups. +// +// head and base are commit hashes or refs. When base is non-empty the log is +// restricted to the range base..head (commits reachable from head but not from +// base), which is the diff-scan contract behind `--since-commit`. The range is +// computed by git itself so that it is independent of commit dates and merge +// topology. An empty base means a full-history scan of head (or of --all when +// head is also empty). An empty head with a non-empty base means every commit +// reachable from any ref but not from base (`--all ^base`), which is what +// `--since-commit X` without `--branch` has always covered; callers that want +// a single-branch diff pass the head explicitly. func (c *Parser) RepoPath( ctx context.Context, source string, head string, - abbreviatedLog bool, + base string, excludedGlobs []string, isBare bool, - additionalArgs ...string, ) (chan *Diff, error) { - args := []string{ - "-C", source, - "log", - "--patch", // https://git-scm.com/docs/git-log#Documentation/git-log.txt---patch - "--full-history", - "--date=iso-strict", - "--pretty=fuller", // https://git-scm.com/docs/git-log#_pretty_formats - "--notes", // https://git-scm.com/docs/git-log#Documentation/git-log.txt---notesltrefgt + args := c.prepGitArgs(source, head, base, excludedGlobs, isBare) + + if c.lowMemoryScan { + return c.repoPathLowMemory(ctx, args) } - if abbreviatedLog { - args = append(args, "--diff-filter=AM") + + showCmd := exec.CommandContext(ctx, + "git", + slices.Concat(args.global, []string{"log"}, args.show, args.log, args.paths)..., + ) + showCmd.Env = args.env + + return c.executeCommand(ctx, showCmd, false) +} + +func (c *Parser) repoPathLowMemory(ctx context.Context, args gitArgs) (chan *Diff, error) { + commitGroups, err := c.enumerateCommits(ctx, args) + if err != nil { + return nil, err } - if head != "" { - args = append(args, head) - } else { - args = append(args, "--all") + + // c.executeCommand returns a channel that is later closed by a + // different goroutine after the command finishes, but we're not + // running a single command anymore. we'll use a channel of channels to + // reduce back to one channel we return to our caller. Unbuffered so + // we have at most one git log running and one git log draining. + diffGroups := make(chan chan *Diff) + go func() { + defer common.RecoverWithExit(ctx) + defer close(diffGroups) + + for group := range commitGroups { + if common.IsDone(ctx) { + return + } + + // `git log` over an explicit list, not `git show`. The two print the + // same thing for ordinary commits, but only log applies --diff-filter + // to whole commits, so this is what keeps the set of scanned commits + // the same as the single-command form. + logCmd := exec.CommandContext(ctx, + "git", + slices.Concat( + args.global, []string{"log"}, args.show, + []string{ + // Keep the commits in the order rev-list gave them. Plain + // --no-walk would re-sort each group by commit date, which + // scrambles the order across groups. + "--no-walk=unsorted", + // Hashes come in on stdin, so the group size is ours to pick. + "--stdin", + }, + args.paths, + )..., + ) + logCmd.Env = args.env + logCmd.Stdin = strings.NewReader(strings.Join(group, "\n") + "\n") + + diffGroup, err := c.executeCommand(ctx, logCmd, false) + if err != nil { + ctx.Logger().Error(err, "Error executing git log for commit group.") + return + } + err = common.CancellableWrite(ctx, diffGroups, diffGroup) + if err != nil { + ctx.Logger().Error(err, "git log iteration cancelled") + return + } + } + }() + + // and this is the single channel we're responsible for returning and + // closing. + diffChan := make(chan *Diff) + go func() { + defer common.RecoverWithExit(ctx) + defer close(diffChan) + + var err error + for groupdiffs := range diffGroups { + for diff := range groupdiffs { + err = common.CancellableWrite(ctx, diffChan, diff) + if err != nil { + return // context cancel + } + } + } + }() + + return diffChan, nil +} + +// enumerateCommits asks git for the hashes of the commits we mean to scan, and +// nothing else. It returns them in groups, so patch generation can start before the +// whole history has been walked. +// +// This uses `git rev-list` rather than `git log`. rev-list is the plumbing command for +// listing commits and never sets up git's diff machinery, which `git log` does as soon +// as a diff option is present. On this repository that is the difference between 43 MB +// and 3.4 MB of peak memory for a phase whose only job is to print hashes, and this is +// the phase that sets the peak on long histories. +// commitGroupSize is how many commits each `git log` gets. +func (c *Parser) commitGroupSize() int { + // A size of zero or less would mean a group that never fills, so it falls back. + if c.groupSize <= 0 { + return logGroupSize + } + return c.groupSize +} + +func (c *Parser) enumerateCommits(ctx context.Context, args gitArgs) (chan []string, error) { + // args.log holds only the options that choose commits, so it can go to rev-list + // as-is. Diff options live in args.show and rev-list would reject them. + cmd := exec.CommandContext(ctx, + "git", slices.Concat( + args.global, []string{"rev-list"}, + args.log, + args.paths, + )...) + cmd.WaitDelay = c.waitDelay + cmd.Env = args.env + + // Keep stderr, because it carries the reason a bad ref or a broken repo failed. + var stderr strings.Builder + cmd.Stderr = &stderr + + stdOut, err := cmd.StdoutPipe() + if err != nil { + return nil, err } - args = append(args, additionalArgs...) // These need to come before -- - for _, glob := range excludedGlobs { - args = append(args, "--", ".", ":(exclude)"+glob) + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to execute git rev-list: %w", err) + } + + // Wait for the first byte before returning. A bad revision makes rev-list fail + // immediately, and this is the last moment we can hand that back to the caller as + // an error. Reporting it any later means closing the channel instead, and a typo in + // a branch name then looks exactly like an empty repository. + reader := bufio.NewReader(stdOut) + if _, err := reader.Peek(1); err != nil { + if waitErr := cmd.Wait(); waitErr != nil { + return nil, fmt.Errorf("failed to list commits: %w: %s", waitErr, strings.TrimSpace(stderr.String())) + } + if !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("failed to read commit list: %w", err) + } + + // git was happy and printed nothing, so there is genuinely nothing to scan. + empty := make(chan []string) + close(empty) + return empty, nil + } + + commitGroups := make(chan []string) + go func() { + defer close(commitGroups) + defer func() { + if err := cmd.Wait(); err != nil { + ctx.Logger().Error(err, "git rev-list exited with error", "stderr", strings.TrimSpace(stderr.String())) + } + }() + + s := bufio.NewScanner(reader) + groupSize := c.commitGroupSize() + commitGroup := make([]string, 0, groupSize) + for s.Scan() && !common.IsDone(ctx) { + commitGroup = append(commitGroup, s.Text()) + if len(commitGroup) < groupSize { + continue + } + if err := common.CancellableWrite(ctx, commitGroups, commitGroup); err != nil { + ctx.Logger().Error(err, "git rev-list stopping early") + return + } + commitGroup = make([]string, 0, groupSize) + } + + // The last group is almost never exactly full. + if len(commitGroup) != 0 { + if err := common.CancellableWrite(ctx, commitGroups, commitGroup); err != nil { + ctx.Logger().Error(err, "failed to flush last commit group") + } + } + + if err := s.Err(); err != nil { + ctx.Logger().Error(err, "error reading commit list") + } + }() + + return commitGroups, nil +} + +func (c *Parser) prepGitArgs(source string, head string, base string, excludedGlobs []string, isBare bool) gitArgs { + // Full-history scans skip deletions; a diff scan must report every change in the range. + abbreviatedLog := base == "" + + args := gitArgs{ + global: []string{ + "-C", source, + }, + log: []string{ + // https://git-scm.com/docs/git-log#Documentation/git-log.txt---full-history + "--full-history", + }, + show: []string{ + // https://git-scm.com/docs/git-show#Documentation/git-show.txt---patch + "--patch", + // https://git-scm.com/docs/git-log#Documentation/git-log.txt---dateformat + "--date=iso-strict", + // https://git-scm.com/docs/git-show#_pretty_formats + "--pretty=fuller", + // https://git-scm.com/docs/git-show#Documentation/git-show.txt---notesref + "--notes", + }, + paths: []string{}, + } + + if abbreviatedLog { + // Only in show. args.log holds the options that choose commits, and it is + // also what the lower-memory scan hands to `git rev-list`, which rejects diff + // options outright. Leaving it out costs nothing: rev-list then lists a few + // commits whose diffs are all filtered away, and the `git log` that generates + // the patches drops those commits itself, exactly as the single-command form + // does. + // https://git-scm.com/docs/git-show#Documentation/git-show.txt---diff-filterACDMRTUXB + args.show = append(args.show, "--diff-filter=AM") + } + + // The positive end of the walk, kept last before the -- (not required but + // sensible). With no head every ref is walked, so a base-only scan is + // `--all ^base`: everything since base on any branch, not just the checked + // out one. The pre-commit hook wants the empty HEAD..HEAD range and passes + // HEAD as both ends itself (see the isPreCommitHook override in main.go). + // https://git-scm.com/docs/git-log#Documentation/git-log.txt---all + switch { + case head != "": + args.log = append(args.log, head) + default: + args.log = append(args.log, "--all") + } + + // `head ^base` is base..head. Keeping the two revisions as separate args + // means head is always the positive end of the range and the base is + // never spliced into a string git has to parse. + // https://git-scm.com/docs/gitrevisions#_specifying_ranges + if base != "" { + args.log = append(args.log, "^"+base) + } + + // Pathspecs live in their own slice so `--` always trails every revision. + if len(excludedGlobs) != 0 { + args.paths = []string{"--", "."} + for _, glob := range excludedGlobs { + // This is not directly doc'd but added in git 1.9.0 and found in pathspec.c + args.paths = append(args.paths, ":(exclude)"+glob) + } } - cmd := exec.CommandContext(ctx, "git", args...) absPath, err := filepath.Abs(source) if err == nil { if !isBare { - cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git")) + args.env = append(args.env, "GIT_DIR="+filepath.Join(absPath, ".git")) } else { - cmd.Env = append(cmd.Env, - "GIT_DIR="+absPath, - ) + args.env = append(args.env, "GIT_DIR="+absPath) // We need those variables to handle incoming commits // while using trufflehog in pre-receive hooks if dir := os.Getenv("GIT_OBJECT_DIRECTORY"); dir != "" { - cmd.Env = append(cmd.Env, "GIT_OBJECT_DIRECTORY="+dir) + args.env = append(args.env, "GIT_OBJECT_DIRECTORY="+dir) } if dir := os.Getenv("GIT_ALTERNATE_OBJECT_DIRECTORIES"); dir != "" { - cmd.Env = append(cmd.Env, "GIT_ALTERNATE_OBJECT_DIRECTORIES="+dir) + args.env = append(args.env, "GIT_ALTERNATE_OBJECT_DIRECTORIES="+dir) } } } - - return c.executeCommand(ctx, cmd, false, c.waitDelay) + return args } // Staged parses the output of the `git diff` command for the `source` path. @@ -301,29 +575,29 @@ func (c *Parser) Staged(ctx context.Context, source string) (chan *Diff, error) cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git")) } - return c.executeCommand(ctx, cmd, true, c.waitDelay) + return c.executeCommand(ctx, cmd, true) } // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete. // waitDelay specifies how long to wait after context cancellation before forcefully killing the process. -func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool, waitDelay time.Duration) (chan *Diff, error) { +func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd, isStaged bool) (chan *Diff, error) { diffChan := make(chan *Diff, 64) stdOut, err := cmd.StdoutPipe() if err != nil { - return diffChan, err + return nil, err } stdErr, err := cmd.StderrPipe() if err != nil { - return diffChan, err + return nil, err } // Set WaitDelay to allow the command additional time to exit after context cancellation - cmd.WaitDelay = waitDelay + cmd.WaitDelay = c.waitDelay err = cmd.Start() if err != nil { - return diffChan, err + return nil, err } go func() { @@ -355,7 +629,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, diffChan chan totalLogSize int ) - var latestState = Initial + latestState := Initial diff := func(c *Commit, opts ...diffOption) *Diff { opts = append(opts, withCustomContentWriter(bufferwriter.New())) @@ -371,10 +645,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, diffChan chan defer common.RecoverWithExit(ctx) defer close(diffChan) - for { - if common.IsDone(ctx) { - break - } + for !common.IsDone(ctx) { line, err := outReader.ReadBytes([]byte("\n")[0]) if err != nil && len(line) == 0 { @@ -616,15 +887,15 @@ func isMergeLine(isStaged bool, latestState ParseState, line []byte) bool { // commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05 func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool { - if isStaged || !(latestState == Initial || - latestState == MessageStartLine || - latestState == MessageEndLine || - latestState == ModeLine || - latestState == IndexLine || - latestState == BinaryFileLine || - latestState == ToFileLine || - latestState == HunkContentLine || - latestState == ParseFailure) { + if isStaged || (latestState != Initial && + latestState != MessageStartLine && + latestState != MessageEndLine && + latestState != ModeLine && + latestState != IndexLine && + latestState != BinaryFileLine && + latestState != ToFileLine && + latestState != HunkContentLine && + latestState != ParseFailure) { return false } @@ -636,7 +907,7 @@ func isCommitLine(isStaged bool, latestState ParseState, line []byte) bool { // Author: Bill Rich func isAuthorLine(isStaged bool, latestState ParseState, line []byte) bool { - if isStaged || !(latestState == CommitLine || latestState == MergeLine) { + if isStaged || (latestState != CommitLine && latestState != MergeLine) { return false } if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) { @@ -692,7 +963,7 @@ func isMessageStartLine(isStaged bool, latestState ParseState, line []byte) bool // Line that starts with 4 spaces func isMessageLine(isStaged bool, latestState ParseState, line []byte) bool { - if isStaged || !(latestState == MessageStartLine || latestState == MessageLine) { + if isStaged || (latestState != MessageStartLine && latestState != MessageLine) { return false } if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) { @@ -726,7 +997,7 @@ func isNotesStartLine(isStaged bool, latestState ParseState, line []byte) bool { // Line after NotesStartLine that starts with 4 spaces func isNotesLine(isStaged bool, latestState ParseState, line []byte) bool { - if isStaged || !(latestState == NotesStartLine || latestState == NotesLine) { + if isStaged || (latestState != NotesStartLine && latestState != NotesLine) { return false } if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) { @@ -748,15 +1019,15 @@ func isNotesEndLine(isStaged bool, latestState ParseState, line []byte) bool { // diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool { - if !(latestState == MessageStartLine || // Empty commit messages can go from MessageStart->Diff - latestState == MessageEndLine || - latestState == NotesEndLine || - latestState == BinaryFileLine || - latestState == ModeLine || - latestState == IndexLine || - latestState == HunkContentLine || - latestState == ParseFailure) { - if !(isStaged && latestState == Initial) { + if latestState != MessageStartLine && + latestState != MessageEndLine && + latestState != NotesEndLine && + latestState != BinaryFileLine && + latestState != ModeLine && + latestState != IndexLine && + latestState != HunkContentLine && + latestState != ParseFailure { + if !isStaged || latestState != Initial { return false } } @@ -774,7 +1045,7 @@ func isDiffLine(isStaged bool, latestState ParseState, line []byte) bool { // rename to new.txt // deleted file mode 100644 func isModeLine(latestState ParseState, line []byte) bool { - if !(latestState == DiffLine || latestState == ModeLine) { + if latestState != DiffLine && latestState != ModeLine { return false } // This could probably be better written. @@ -793,7 +1064,7 @@ func isModeLine(latestState ParseState, line []byte) bool { // index 1ed6fbee1..aea1e643a 100644 // index 00000000..e69de29b func isIndexLine(latestState ParseState, line []byte) bool { - if !(latestState == DiffLine || latestState == ModeLine) { + if latestState != DiffLine && latestState != ModeLine { return false } if len(line) > 6 && bytes.Equal(line[:6], []byte("index ")) { @@ -848,7 +1119,7 @@ func pathFromBinaryLine(line []byte) (string, bool) { // --- a/internal/addrs/move_endpoint_module.go // --- /dev/null func isFromFileLine(latestState ParseState, line []byte) bool { - if !(latestState == IndexLine || latestState == ModeLine) { + if latestState != IndexLine && latestState != ModeLine { return false } if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) { @@ -906,7 +1177,7 @@ func pathFromToFileLine(line []byte) (string, bool) { // @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int) func isHunkLineNumberLine(latestState ParseState, line []byte) bool { - if !(latestState == ToFileLine || latestState == HunkContentLine) { + if latestState != ToFileLine && latestState != HunkContentLine { return false } if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) { @@ -918,7 +1189,7 @@ func isHunkLineNumberLine(latestState ParseState, line []byte) bool { // fmt.Println("ok") // (There's a space before `fmt` that gets removed by the formatter.) func isHunkContextLine(latestState ParseState, line []byte) bool { - if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) { + if latestState != HunkLineNumberLine && latestState != HunkContentLine { return false } if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) { @@ -929,7 +1200,7 @@ func isHunkContextLine(latestState ParseState, line []byte) bool { // +fmt.Println("ok") func isHunkPlusLine(latestState ParseState, line []byte) bool { - if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) { + if latestState != HunkLineNumberLine && latestState != HunkContentLine { return false } if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) { @@ -940,7 +1211,7 @@ func isHunkPlusLine(latestState ParseState, line []byte) bool { // -fmt.Println("ok") func isHunkMinusLine(latestState ParseState, line []byte) bool { - if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) { + if latestState != HunkLineNumberLine && latestState != HunkContentLine { return false } if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) { @@ -965,7 +1236,7 @@ func isHunkNewlineWarningLine(latestState ParseState, line []byte) bool { // // commit 00920984e3435057f09cee5468850f7546dfa637 (tag: v3.42.0) func isHunkEmptyLine(latestState ParseState, line []byte) bool { - if !(latestState == HunkLineNumberLine || latestState == HunkContentLine) { + if latestState != HunkLineNumberLine && latestState != HunkContentLine { return false } // TODO: Can this also be `\n\r`? @@ -993,10 +1264,25 @@ func cleanupParse(ctx context.Context, currentCommit *Commit, currentDiff *Diff, if currentDiff != nil && (currentDiff.Len() > 0 || currentDiff.IsBinary) { currentDiff.Commit = currentCommit diffChan <- currentDiff - } - if currentCommit != nil { - if totalLogSize != nil { - *totalLogSize += currentCommit.Size + if currentCommit != nil { + currentCommit.hasDiffs = true } } + + if currentCommit == nil { + return + } + if totalLogSize != nil { + *totalLogSize += currentCommit.Size + } + + // A commit is normally finished off in the loop above, when the next commit line + // shows up. The last commit in the stream never gets that, so it is finished here + // instead, and it needs the same rule: a commit with no diffs of its own still has + // a message, an author and notes worth scanning, so it goes out on its own. + // + // Staged diffs come through here too and carry no commit, hence the hash check. + if !currentCommit.hasDiffs && currentCommit.Hash != "" { + diffChan <- &Diff{Commit: currentCommit} + } } diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.go index e436c25fb376..141cf0832970 100644 --- a/pkg/gitparse/gitparse_test.go +++ b/pkg/gitparse/gitparse_test.go @@ -2,6 +2,8 @@ package gitparse import ( "bytes" + "path/filepath" + "slices" "strings" "testing" "time" @@ -17,6 +19,103 @@ import ( bufferedfilewriter "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffered_file_writer" ) +func TestPrepGitArgs(t *testing.T) { + t.Setenv("GIT_OBJECT_DIRECTORY", "") + t.Setenv("GIT_ALTERNATE_OBJECT_DIRECTORIES", "") + repopath := t.TempDir() + + p := Parser{} + + // Full history of everything: no head, no base. + args := p.prepGitArgs(repopath, "", "", nil, false) + assert.Equal(t, []string{"-C", repopath}, args.global) + assert.Contains(t, args.log, "--all") + // A full-history scan skips deletions; a diff scan must not. The filter is a + // diff option, so it lives in show and never reaches rev-list via args.log. + assert.NotContains(t, args.log, "--diff-filter=AM") + assert.Contains(t, args.show, "--diff-filter=AM") + assertNoRangeExclusion(t, args.log) + assert.Equal(t, []string{"GIT_DIR=" + filepath.Join(repopath, ".git")}, args.env) + assert.Empty(t, args.paths) + + // Full history of one branch, with excludes, in a bare repo. + args = p.prepGitArgs(repopath, "branchname", "", []string{"some/file.txt", "bloated.dat"}, true) + // head + assert.Contains(t, args.log, "branchname") + assert.NotContains(t, args.log, "--all") + // abbreviatedLog is implied by the empty base. Only show carries the diff + // filter: args.log holds the options that choose commits and is what the + // lower-memory scan gives to `git rev-list`, which rejects diff options. + assert.NotContains(t, args.log, "--diff-filter=AM") + assert.Contains(t, args.show, "--diff-filter=AM") + assertNoRangeExclusion(t, args.log) + // excludedGlobs + assert.Contains(t, args.paths, "--") + assert.Contains(t, args.paths, ":(exclude)bloated.dat") + // isBare + assert.Equal(t, []string{"GIT_DIR=" + repopath}, args.env) + + // Diff scan base..head: git must receive the exclusion itself so the range + // does not depend on commit-date ordering (INT-1054). + args = p.prepGitArgs(repopath, "headsha", "basesha", nil, false) + assert.Contains(t, args.log, "headsha") + assert.Contains(t, args.log, "^basesha") + assert.NotContains(t, args.log, "--all") + assert.NotContains(t, args.log, "--diff-filter=AM") + // git show is run per explicit hash in the low-memory path and must never + // be handed a range. + assert.NotContains(t, args.show, "--diff-filter=AM") + assert.NotContains(t, args.show, "^basesha") + // Revision args follow the flags; the exclusion belongs with the revisions. + assert.Greater(t, indexOf(args.log, "^basesha"), indexOf(args.log, "headsha")) + + // A range with excludes: the pathspec separator must land after both + // revisions or git reads the exclusion as a path. + args = p.prepGitArgs(repopath, "headsha", "basesha", []string{"bloated.dat"}, false) + full := slices.Concat(args.log, args.paths) + assert.Greater(t, indexOf(full, "--"), indexOf(full, "^basesha")) + + // A base with no head is `--all ^base`: everything since the base on any + // ref, which is what `--since-commit X` without `--branch` covers. The + // parser must not invent a head here; narrowing to HEAD would silently + // drop commits that live only on other branches. + args = p.prepGitArgs(repopath, "", "basesha", nil, false) + assert.Contains(t, args.log, "--all") + assert.Contains(t, args.log, "^basesha") + assert.NotContains(t, args.log, "HEAD") + assert.NotContains(t, args.log, "--diff-filter=AM") + assert.Greater(t, indexOf(args.log, "^basesha"), indexOf(args.log, "--all")) + + // test env passthrough used for pre-receive + t.Setenv("GIT_OBJECT_DIRECTORY", "foo") + t.Setenv("GIT_ALTERNATE_OBJECT_DIRECTORIES", "bar") + + args = p.prepGitArgs(repopath, "", "", nil, true) + assert.Equal(t, []string{ + "GIT_DIR=" + repopath, + "GIT_OBJECT_DIRECTORY=foo", + "GIT_ALTERNATE_OBJECT_DIRECTORIES=bar", + }, args.env) +} + +// assertNoRangeExclusion fails if any revision argument excludes commits +// (a leading "^"), which must only happen for diff scans with a base. +func assertNoRangeExclusion(t *testing.T, logArgs []string) { + t.Helper() + for _, a := range logArgs { + assert.False(t, strings.HasPrefix(a, "^"), "unexpected range exclusion %q in %v", a, logArgs) + } +} + +func indexOf(args []string, want string) int { + for i, a := range args { + if a == want { + return i + } + } + return -1 +} + type testCaseLine struct { latestState ParseState line []byte @@ -750,7 +849,6 @@ func TestToFileLinePathParse(t *testing.T) { // `asserts` on `Diff`'s _structure_, giving better test output than just comparing two // diffs. func assertDiffEqualToExpected(t *testing.T, expected *Diff, actual *Diff) { - // Use `cmp.Diff` to automatically compare all the exported fields. This allows this test to grow automatically if // new exported fields are added to these structs. However, the most important field we want to test is unexpected // (i.e. contentWriter) which is where the actual content of the diff is stored. We break this out next. @@ -1476,8 +1574,8 @@ func TestMaxCommitSize(t *testing.T) { if diffCount != 2 { t.Errorf("Commit count does not match. Got: %d, expected: %d", diffCount, 2) } - } + func TestWaitDelay(t *testing.T) { // Test that WithWaitDelay sets the waitDelay correctly customDelay := 10 * time.Second diff --git a/pkg/gitparse/lowmemory_test.go b/pkg/gitparse/lowmemory_test.go new file mode 100644 index 000000000000..f4220719caba --- /dev/null +++ b/pkg/gitparse/lowmemory_test.go @@ -0,0 +1,296 @@ +package gitparse + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" +) + +// The lower-memory scan splits one `git log` into a `git rev-list` that lists commits +// and a `git log` per group that generates their patches. These tests hold it to the +// only thing that really matters: it must find exactly what the single-command form +// finds, wherever the group boundaries happen to land. + +// collectDiffs drains a diff channel into something comparable. Content is included, +// since a group boundary in the wrong place could keep the commit and lose its patch. +func collectDiffs(t *testing.T, diffChan chan *Diff) []string { + t.Helper() + + var out []string + for diff := range diffChan { + content := "" + // A commit with no diffs arrives with nothing written, and asking such a diff + // for its content fails, so check before reading. + if diff.contentWriter != nil && diff.Len() > 0 { + got, err := diff.contentWriter.String() + if err != nil { + t.Fatalf("reading diff content: %v", err) + } + content = got + } + out = append(out, strings.Join([]string{diff.Commit.Hash, diff.PathB, content}, "\x00")) + } + return out +} + +func TestLowMemoryScanMatchesSingleProcess(t *testing.T) { + repo := testRepoRoot(t) + + // abbreviatedLog is the caller's BaseHash == "". Only the empty base runs here: + // the checkout may be shallow, so the real history has no reliable base commit. + // Diff scans with a base go through the lower-memory path in the pkg/sources/git + // range tests, which build their own fixtures. + // + // The group sizes go down to 1 on purpose. Every group ends a stream, and a commit + // with no diffs used to be dropped when it landed at the end of one, so a size of 1 + // puts every commit in that spot at once. Before cleanupParse learned to finish off + // the last commit, this repository lost 84 diffs at size 1 and 1 at size 75. + for _, tc := range []struct { + name string + base string + }{ + {"abbreviated", ""}, + } { + base, name := tc.base, tc.name + + t.Run(name, func(t *testing.T) { + ctx := context.Background() + + single := NewParser() + singleChan, err := single.RepoPath(ctx, repo, "", base, nil, false) + if err != nil { + t.Fatalf("single-process RepoPath: %v", err) + } + want := collectDiffs(t, singleChan) + if len(want) == 0 { + t.Fatal("single-process scan produced no diffs") + } + + // Group sizes small enough that this repository crosses boundaries, which + // is the only place the two forms can drift apart. + for _, groupSize := range []int{1, 2, 7, 500} { + low := NewParser(UseLowMemoryScan()) + low.groupSize = groupSize + + lowChan, err := low.RepoPath(ctx, repo, "", base, nil, false) + if err != nil { + t.Fatalf("group size %d: RepoPath: %v", groupSize, err) + } + got := collectDiffs(t, lowChan) + + if len(got) != len(want) { + t.Fatalf("group size %d: got %d diffs, want %d", groupSize, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("group size %d: diff %d differs\n got: %q\nwant: %q", + groupSize, i, got[i], want[i]) + } + } + } + }) + } +} + +// TestLowMemoryScanExcludedGlobs checks the path filters reach both commands. They +// decide which commits rev-list lists and which files the patches contain, so sending +// them to only one of the two would quietly change what gets scanned. +func TestLowMemoryScanExcludedGlobs(t *testing.T) { + repo := testRepoRoot(t) + ctx := context.Background() + globs := []string{"*.go"} + + single := NewParser() + singleChan, err := single.RepoPath(ctx, repo, "", "", globs, false) + if err != nil { + t.Fatalf("single-process RepoPath: %v", err) + } + want := collectDiffs(t, singleChan) + + low := NewParser(UseLowMemoryScan()) + low.groupSize = 13 + lowChan, err := low.RepoPath(ctx, repo, "", "", globs, false) + if err != nil { + t.Fatalf("low-memory RepoPath: %v", err) + } + got := collectDiffs(t, lowChan) + + if len(got) != len(want) { + t.Fatalf("got %d diffs, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("diff %d differs\n got: %q\nwant: %q", i, got[i], want[i]) + } + } +} + +// TestLowMemoryScanUnknownHead covers the error the old form threw away. A bad revision +// used to be logged and the channel closed, so a typo in a branch name looked exactly +// like an empty repository. +func TestLowMemoryScanUnknownHead(t *testing.T) { + repo := testRepoRoot(t) + + parser := NewParser(UseLowMemoryScan()) + if _, err := parser.RepoPath(context.Background(), repo, "no-such-ref-exists", "", nil, false); err == nil { + t.Fatal("expected an error for an unknown head, got nil") + } +} + +// TestLowMemoryScanEmptyRepo checks a repository with no commits ends the scan cleanly +// rather than failing. rev-list prints nothing and exits zero, which has to be told +// apart from rev-list printing nothing because it failed. +func TestLowMemoryScanEmptyRepo(t *testing.T) { + dir := t.TempDir() + runTestGit(t, dir, "init", "-q") + + parser := NewParser(UseLowMemoryScan()) + diffChan, err := parser.RepoPath(context.Background(), dir, "", "", nil, false) + if err != nil { + t.Fatalf("RepoPath on an empty repo: %v", err) + } + if n := len(collectDiffs(t, diffChan)); n != 0 { + t.Fatalf("got %d diffs from an empty repo, want 0", n) + } +} + +// TestLowMemoryScanAbandonedByConsumer covers a caller that stops reading early, which +// is what a depth-limited scan does. Reading one diff and walking away must not leave +// the scan wedged; the cancel in ScanCommits is what releases it in production, so the +// same cancel stands in for it here. +func TestLowMemoryScanAbandonedByConsumer(t *testing.T) { + repo := testRepoRoot(t) + ctx, cancel := context.WithCancel(context.Background()) + + parser := NewParser(UseLowMemoryScan()) + parser.groupSize = 2 + + diffChan, err := parser.RepoPath(ctx, repo, "", "", nil, false) + if err != nil { + t.Fatalf("RepoPath: %v", err) + } + + // Take a single diff, then stop, the way the scan loop does at max depth. + if _, ok := <-diffChan; !ok { + t.Fatal("expected at least one diff") + } + cancel() + + // Draining to the end must finish. If it hangs the test times out, which is the + // failure we are looking for. + for range diffChan { //nolint:revive // draining + } +} + +func TestCommitGroupSize(t *testing.T) { + // A size of zero or less would mean a group that never fills, so it falls back. + for _, tc := range []struct{ set, want int }{ + {0, logGroupSize}, + {-1, logGroupSize}, + {1, 1}, + {99, 99}, + } { + parser := NewParser() + parser.groupSize = tc.set + if got := parser.commitGroupSize(); got != tc.want { + t.Errorf("groupSize %d: got %d, want %d", tc.set, got, tc.want) + } + } +} + +// testRepoRoot returns the root of the git checkout the tests run from. The real +// history has the merges, binaries, renames and uneven commit sizes that a hand-built +// fixture does not, which is where a batching mistake actually shows up. +func testRepoRoot(tb testing.TB) string { + tb.Helper() + + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + tb.Skip("not running from a git checkout") + } + root := strings.TrimSpace(string(out)) + if _, err := os.Stat(filepath.Join(root, ".git")); err != nil { + tb.Skip("no .git directory") + } + return root +} + +func runTestGit(tb testing.TB, dir string, args ...string) { + tb.Helper() + + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + tb.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } +} + +func testGitOutput(tb testing.TB, dir string, args ...string) string { + tb.Helper() + + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.Output() + if err != nil { + tb.Fatalf("git %s: %v", strings.Join(args, " "), err) + } + return string(out) +} + +// TestTrailingCommitWithoutDiffsIsReported pins down the rule that a commit is finished +// off at the end of a stream, not only when the next commit line arrives. +// +// A commit with no diffs of its own still carries a message, an author and notes, and +// those are worth scanning. The main loop has always sent such a commit on when it saw +// the next one start, but the end of the stream had no equivalent, so whichever commit +// happened to be last was dropped. One `git log` for a whole repository made that nearly +// harmless, since only the oldest commit in the history was ever in that spot. Splitting +// the log into groups puts a different commit there for every group. +func TestTrailingCommitWithoutDiffsIsReported(t *testing.T) { + dir := t.TempDir() + runTestGit(t, dir, "init", "-q") + runTestGit(t, dir, "config", "user.email", "test@example.com") + runTestGit(t, dir, "config", "user.name", "Test") + + if err := os.WriteFile(filepath.Join(dir, "f.txt"), []byte("hello\n"), 0o600); err != nil { + t.Fatal(err) + } + runTestGit(t, dir, "add", "-A") + runTestGit(t, dir, "commit", "-qm", "adds a file") + + // An empty commit has a message but changes nothing, which is the shape that used + // to disappear. + runTestGit(t, dir, "commit", "-q", "--allow-empty", "-m", "empty commit worth scanning") + + // Group size 1 puts the empty commit at the end of its own stream, which is exactly + // where it used to be lost. A real base keeps the scan unabbreviated; without one + // the diff filter would drop the empty commit for an unrelated reason. + parser := NewParser(UseLowMemoryScan()) + parser.groupSize = 1 + base := strings.TrimSpace(testGitOutput(t, dir, "rev-parse", "HEAD~1")) + + diffChan, err := parser.RepoPath(context.Background(), dir, "", base, nil, false) + if err != nil { + t.Fatalf("RepoPath: %v", err) + } + + var messages []string + for diff := range diffChan { + messages = append(messages, diff.Commit.Message.String()) + } + + var found bool + for _, msg := range messages { + if strings.Contains(msg, "empty commit worth scanning") { + found = true + break + } + } + if !found { + t.Fatalf("the empty commit was never reported; got %d diffs with messages %q", len(messages), messages) + } +} diff --git a/pkg/giturl/giturl.go b/pkg/giturl/giturl.go index ad8574bb0f7b..aebbdf553dfb 100644 --- a/pkg/giturl/giturl.go +++ b/pkg/giturl/giturl.go @@ -1,9 +1,9 @@ package giturl import ( + regexp "github.com/wasilibs/go-re2" "net/url" "path/filepath" - "regexp" "strconv" "strings" @@ -219,9 +219,9 @@ var ( // UpdateLinkLineNumber updates the line number in a repository link. // Used post-link generation to refine reported issue locations within large scanned blocks. func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) string { - link = strings.Replace(link, "%", "%25", -1) - link = strings.Replace(link, "[", "%5B", -1) - link = strings.Replace(link, "]", "%5D", -1) + link = strings.ReplaceAll(link, "%", "%25") + link = strings.ReplaceAll(link, "[", "%5B") + link = strings.ReplaceAll(link, "]", "%5D") parsedURL, err := url.Parse(link) if err != nil { ctx.Logger().Error(err, "unable to parse link to update line number", "link", link) diff --git a/pkg/handlers/apk.go b/pkg/handlers/apk.go index 858161e53a95..2151f3ad9058 100644 --- a/pkg/handlers/apk.go +++ b/pkg/handlers/apk.go @@ -6,9 +6,9 @@ import ( "encoding/xml" "errors" "fmt" + regexp "github.com/wasilibs/go-re2" "io" "path/filepath" - "regexp" "strings" "sync" "time" @@ -173,7 +173,24 @@ func (h *apkHandler) HandleFile(ctx logContext.Context, input fileReader) chan D }() start := time.Now() - err := h.processAPK(ctx, input, apkChan) + + zipReader, err := createZipReader(input) + if err != nil { + h.measureLatencyAndHandleErrors(ctx, start, err, apkChan) + return + } + + resTable, err := parseResTable(zipReader) + if err != nil { + h.measureLatencyAndHandleErrors(ctx, start, err, apkChan) + return + } + + if pkgName := extractPackageName(zipReader, resTable); pkgName != "" { + ctx = logContext.WithValues(ctx, "apk_package", pkgName) + } + + err = h.processAPK(ctx, zipReader, resTable, apkChan) if err == nil { h.metrics.incFilesProcessed() } @@ -185,19 +202,7 @@ func (h *apkHandler) HandleFile(ctx logContext.Context, input fileReader) chan D } // processAPK processes the apk file and sends the extracted data to the provided channel. -func (h *apkHandler) processAPK(ctx logContext.Context, input fileReader, apkChan chan DataOrErr) error { - // Create a ZIP reader from the input fileReader - zipReader, err := createZipReader(input) - if err != nil { - return err - } - - // Extract the resources.arsc file into a ResourceTable (needed for XML decoding) - resTable, err := parseResTable(zipReader) - if err != nil { - return err - } - +func (h *apkHandler) processAPK(ctx logContext.Context, zipReader *zip.Reader, resTable *apkparser.ResourceTable, apkChan chan DataOrErr) error { // Process the ResourceTable file for secrets if err := h.processResources(ctx, resTable, apkChan); err != nil { ctx.Logger().Error(err, "failed to process resources.arsc") @@ -206,7 +211,7 @@ func (h *apkHandler) processAPK(ctx logContext.Context, input fileReader, apkCha // Process all files for secrets for _, file := range zipReader.File { if err := h.processFile(ctx, file, resTable, apkChan); err != nil { - ctx.Logger().V(2).Info(fmt.Sprintf("failed to process file: %s", file.Name), "error", err) + ctx.Logger().V(2).Info("failed to process file", "file", file.Name, "error", err) } } return nil @@ -217,10 +222,8 @@ func (h *apkHandler) processResources(ctx logContext.Context, resTable *apkparse if resTable == nil { return errors.New("ResourceTable is nil") } - rscStrRdr, err := extractStringsFromResTable(resTable) - if err != nil { - return fmt.Errorf("failed to parse strings from resources.arsc: %w", err) - } + provider := &apkResourceTable{table: resTable} + rscStrRdr := extractStringsFromResTable(provider) return h.handleAPKFileContent(ctx, rscStrRdr, "resources.arsc", apkChan) } @@ -241,10 +244,10 @@ func (h *apkHandler) processFile( if err != nil { return fmt.Errorf("failed to read file %s: %w", file.Name, err) } - defer f.Close() + defer func() { _ = f.Close() }() rdr := iobuf.NewBufferedReaderSeeker(f) - defer rdr.Close() + defer func() { _ = rdr.Close() }() var contentReader io.Reader // Decode the file based on its extension @@ -255,7 +258,7 @@ func (h *apkHandler) processFile( return fmt.Errorf("failed to decode xml file %s: %w", file.Name, err) } case ".dex": - contentReader, err = h.processDexFile(ctx, rdr) + contentReader, err = h.processDexFile(ctx, rdr, file.Name) if err != nil { return fmt.Errorf("failed to decode dex file %s: %w", file.Name, err) } @@ -306,7 +309,7 @@ func parseResTable(zipReader *zip.Reader) (*apkparser.ResourceTable, error) { } resTable, err := apkparser.ParseResourceTable(rdr) - rdr.Close() + _ = rdr.Close() if err != nil { return nil, err } @@ -326,55 +329,98 @@ func openFile(file *zip.File) (io.ReadCloser, error) { return rc, nil } -// extractStringsFromResTable extracts the strings from the resources table -// Note: This is a hacky way to get the strings from the resources table -// APK strings are typically (always?) stored in the 0x7f000000-0x7fffffff range +// resourceEntryData holds the extracted fields from a single resource table entry. +type resourceEntryData struct { + Key string + ResourceType string + Value string +} + +// GetEntryer abstracts resource table lookups for testability. +type GetEntryer interface { + GetEntry(id uint32) (*resourceEntryData, error) +} + +// apkResourceTable adapts *apkparser.ResourceTable to the GetEntryer interface. +type apkResourceTable struct { + table *apkparser.ResourceTable +} + +// GetEntry retrieves a resource entry by ID, returning nil if the entry does not exist. +func (t *apkResourceTable) GetEntry(id uint32) (*resourceEntryData, error) { + entry, _ := t.table.GetResourceEntry(id) + if entry == nil { + return nil, nil + } + val, err := entry.GetValue().String() + return &resourceEntryData{ + Key: entry.Key, + ResourceType: entry.ResourceType, + Value: val, + }, err +} + +// extractStringsFromResTable extracts the strings from the resources table, which is provided by the +// `GetEntryer` interface (a thin wrapper around `apkparser.ResourceTable` for testability). +// APK strings are typically stored in the 0x7f000000-0x7fffffff range. // https://chromium.googlesource.com/chromium/src/+/master/build/android/docs/life_of_a_resource.md -func extractStringsFromResTable(resTable *apkparser.ResourceTable) (io.Reader, error) { +func extractStringsFromResTable(provider GetEntryer) io.Reader { var resourceStrings bytes.Buffer inStrings := false for i := 0x7f000000; i <= 0x7fffffff; i++ { - entry, _ := resTable.GetResourceEntry(uint32(i)) + entry, err := provider.GetEntry(uint32(i)) if entry == nil { continue } if entry.ResourceType == "string" { - inStrings = true - val, err := entry.GetValue().String() if err != nil { - return nil, err + continue } - // Write directly to the buffer + inStrings = true resourceStrings.WriteString(entry.Key) resourceStrings.WriteString(": ") - resourceStrings.WriteString(val) + resourceStrings.WriteString(entry.Value) resourceStrings.WriteString("\n") } - // Exit the loop if we've finished processing the strings + // The contiguity assumption is valid for standard AAPT output, but we might want to revisit this if there's + // shared libs or multiple packages in a group. if inStrings && entry.ResourceType != "string" { break } } - return &resourceStrings, nil + return &resourceStrings } // processDexFile decodes the dex file and returns the relevant instructions -func (h *apkHandler) processDexFile(ctx logContext.Context, rdr io.ReaderAt) (io.Reader, error) { +func (h *apkHandler) processDexFile(ctx logContext.Context, rdr io.ReaderAt, dexFile string) (io.Reader, error) { dexReader, err := dextk.Read(rdr, dextk.WithReadCache(16)) if err != nil { return nil, err } - // Get relevant instruction data from the dex file var dexOutput bytes.Buffer - ci := dexReader.ClassIter() - for ci.HasNext() { - node, err := ci.Next() - if err != nil { - ctx.Logger().Error(err, "failed to process a dex class") - break - } - h.processDexClass(ctx, dexReader, node, &dexOutput) + var classErrors int + // Use an index-based loop instead of ClassIter to guarantee progress. + // The iterator's pos field only advances after a successful `ReadClassAndParse`, so a panic would leave it stuck + // on the same class and the HasNext loop would spin forever. + for id := uint32(0); id < dexReader.ClassDefCount; id++ { + func() { + defer func() { + if r := recover(); r != nil { + classErrors++ + } + }() + + node, err := dexReader.ReadClassAndParse(id) + if err != nil { + classErrors++ + return + } + h.processDexClass(ctx, dexReader, node, &dexOutput) + }() + } + if classErrors > 0 { + ctx.Logger().V(2).Info("skipped malformed dex classes", "dex_file", dexFile, "skipped_classes", classErrors) } return &dexOutput, nil @@ -387,6 +433,12 @@ func (h *apkHandler) processDexClass( node dextk.ClassNode, dexOutput *bytes.Buffer, ) { + defer func() { + if r := recover(); r != nil { + ctx.Logger().V(2).Info("panic processing dex class", "class", node.Name.Parsed, "error", fmt.Sprintf("%v", r)) + } + }() + var classOutput bytes.Buffer methodValues := make(map[string]struct{}) @@ -428,20 +480,38 @@ func processDexMethod( } } -// parseDexInstructions processes a dex method and returns the string representation of the instruction -func parseDexInstructions(r *dextk.Reader, m dextk.MethodNode, methodValues map[string]struct{}) (*bytes.Buffer, error) { +// parseDexInstructions processes a dex method and returns the string representation of the instruction. +func parseDexInstructions(r *dextk.Reader, m dextk.MethodNode, methodValues map[string]struct{}) (buf *bytes.Buffer, err error) { var instrBuf bytes.Buffer if m.CodeOff == 0 { return &instrBuf, nil } + // This deferred recovery catches panics from dextk which can occur with obfuscated APKs. + // This ensures it keeps processing and returns a valid *bytes.Buffer pointer even when a panic occurs. + // Specifically, a panic like `runtime error: slice bounds out of range [65536:16384]` becomes a normal error + // return. The caller (processDexMethod) already handles errors gracefully. It logs and moves on to the next + // method. Without this recovery, the panic would propagate up and abort the entire APK. + // Finally, it returns a valid non-zero *bytes.Buffer even when a panic occurs. + defer func() { + if r := recover(); r != nil { + if e, ok := r.(error); ok { + err = e + } else { + err = fmt.Errorf("%v", r) + } + buf = &instrBuf + } + }() + c, err := r.ReadCodeAndParse(m.CodeOff) if err != nil { return &instrBuf, err } - // Iterate over the instructions and extract the relevant values + // Iterate over the instructions and extract possible secrets into `methodValues` if the instruction type is + // `const-string`. for _, o := range c.Ops { oStr := o.String() @@ -505,3 +575,52 @@ func decodeXML(rdr io.ReadSeeker, resTable *apkparser.ResourceTable) (io.Reader, } return nil, err } + +// extractPackageName attempts to extract the Android package name from AndroidManifest.xml. +// Returns an empty string if the manifest is missing or the package attribute cannot be parsed. +// This is only for logging the package name, however it can be extremely difficult to debug trufflehog when running +// against gigabytes of APKs. You will want this metadata in the error messages. +func extractPackageName(zipReader *zip.Reader, resTable *apkparser.ResourceTable) (pkgName string) { + defer func() { + if r := recover(); r != nil { + pkgName = "" + } + }() + + for _, file := range zipReader.File { + if file.Name != "AndroidManifest.xml" { + continue + } + rc, err := openFile(file) + if err != nil { + return "" + } + defer func() { _ = rc.Close() }() + + rdr := iobuf.NewBufferedReaderSeeker(rc) + defer func() { _ = rdr.Close() }() + + decoded, err := decodeXML(rdr, resTable) + if err != nil { + return "" + } + + decoder := xml.NewDecoder(decoded) + for { + tok, err := decoder.Token() + if err != nil { + break + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "manifest" { + for _, attr := range se.Attr { + if attr.Name.Local == "package" { + return attr.Value + } + } + break + } + } + break + } + return "" +} diff --git a/pkg/handlers/apk_content_detection_test.go b/pkg/handlers/apk_content_detection_test.go new file mode 100644 index 000000000000..c8da1021b0da --- /dev/null +++ b/pkg/handlers/apk_content_detection_test.go @@ -0,0 +1,116 @@ +package handlers + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" +) + +// makeAPKZip builds an in-memory zip that satisfies isAPKFile (contains both +// AndroidManifest.xml and classes.dex). A large stored padding entry is written +// first so the manifest/dex local headers fall outside mimetype's detection +// window, keeping mimetype's verdict "application/zip" rather than APK. This +// forces the content-based isAPKFile deep scan (rather than mimetype's own APK +// detection) to be what promotes the file to the APK handler. +func makeAPKZip(t *testing.T, extraFiles map[string][]byte) []byte { + t.Helper() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + padding, err := zw.CreateHeader(&zip.FileHeader{Name: "padding.txt", Method: zip.Store}) + require.NoError(t, err) + _, err = padding.Write(bytes.Repeat([]byte("A"), 5000)) + require.NoError(t, err) + + for _, name := range []string{"AndroidManifest.xml", "classes.dex"} { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte("placeholder")) + require.NoError(t, err) + } + + for name, content := range extraFiles { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write(content) + require.NoError(t, err) + } + + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// makeNonAPKZip builds a plain in-memory zip without any APK markers. +func makeNonAPKZip(t *testing.T) []byte { + t.Helper() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("hello.txt") + require.NoError(t, err) + _, err = w.Write([]byte("just some content")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// TestNewFileReaderAPKContentDetection verifies APK detection is driven by file +// content (the presence of AndroidManifest.xml + classes.dex) and not by any +// filename/extension: the reader is promoted to the APK mime type with no +// extension supplied, while a zip lacking those markers stays a generic zip. +func TestNewFileReaderAPKContentDetection(t *testing.T) { + feature.EnableAPKHandler.Store(true) + t.Cleanup(func() { feature.EnableAPKHandler.Store(false) }) + + ctx := context.Background() + + rdr, err := newFileReader(ctx, bytes.NewReader(makeAPKZip(t, nil))) + require.NoError(t, err) + assert.Equal(t, string(apkMime), rdr.mime.String(), "APK content should route to the APK handler without an extension") + require.NoError(t, rdr.Close()) + + rdrPlain, err := newFileReader(ctx, bytes.NewReader(makeNonAPKZip(t))) + require.NoError(t, err) + assert.Equal(t, string(zipMime), rdrPlain.mime.String(), "a zip without APK markers should remain a generic zip") + require.NoError(t, rdrPlain.Close()) +} + +// TestNewFileReaderAPKDetectionDisabled ensures APK content detection does not +// run when the feature flag is off, leaving the file as a generic zip. +func TestNewFileReaderAPKDetectionDisabled(t *testing.T) { + feature.EnableAPKHandler.Store(false) + + rdr, err := newFileReader(context.Background(), bytes.NewReader(makeAPKZip(t, nil))) + require.NoError(t, err) + assert.Equal(t, string(zipMime), rdr.mime.String()) + require.NoError(t, rdr.Close()) +} + +// TestNewFileReaderAPKCheckErrorFallsThrough ensures that when the APK feature +// flag is on and mimetype identifies a file as a zip, but the content-based APK +// check fails to parse it (e.g. truncated/corrupted zip, or a polyglot that +// mimetype still reports as zip), newFileReader does NOT return a fatal error. +// Instead it must fall through to normal handling so the file is still processed +// rather than skipped entirely. This guards the regression where dropping the +// .apk extension guard made every unparseable zip/jar unprocessable. +func TestNewFileReaderAPKCheckErrorFallsThrough(t *testing.T) { + feature.EnableAPKHandler.Store(true) + t.Cleanup(func() { feature.EnableAPKHandler.Store(false) }) + + // Start from a valid zip (so mimetype detects "application/zip") then truncate + // the tail, removing the end-of-central-directory record so zip.NewReader fails. + valid := makeNonAPKZip(t) + truncated := valid[:len(valid)-10] + + rdr, err := newFileReader(context.Background(), bytes.NewReader(truncated)) + require.NoError(t, err, "APK check failure must not be fatal; file should fall through to normal handling") + assert.Equal(t, string(zipMime), rdr.mime.String(), "file should remain a generic zip, not be skipped") + require.NoError(t, rdr.Close()) +} diff --git a/pkg/handlers/apk_test.go b/pkg/handlers/apk_test.go index b8ca00530a0e..fa9ce45f9304 100644 --- a/pkg/handlers/apk_test.go +++ b/pkg/handlers/apk_test.go @@ -1,12 +1,13 @@ package handlers import ( + regexp "github.com/wasilibs/go-re2" "io" "net/http" - "regexp" "strings" "testing" + "github.com/avast/apkparser" "github.com/stretchr/testify/assert" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -20,7 +21,7 @@ func TestAPKHandler(t *testing.T) { matchString string }{ "apk_with_3_leaked_keys": { - archiveURL: "https://raw.githubusercontent.com/trufflesecurity/leakyAPK/main/aws_leak.apk", + archiveURL: "https://raw.githubusercontent.com/trufflesecurity/trufflehog-test-assets/main/aws_leak.apk", expectedChunks: 942, // Note: the secret count is 4 instead of 3 b/c we're not actually running the secret detection engine, // we're just looking for a string match. There is one extra string match in the APK (but only 3 detected secrets). @@ -34,7 +35,7 @@ func TestAPKHandler(t *testing.T) { resp, err := http.Get(testCase.archiveURL) assert.NoError(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() handler := newAPKHandler() @@ -42,7 +43,7 @@ func TestAPKHandler(t *testing.T) { if err != nil { t.Errorf("error creating reusable reader: %s", err) } - defer newReader.Close() + defer func() { _ = newReader.Close() }() archiveChan := handler.HandleFile(context.Background(), newReader) @@ -72,15 +73,14 @@ func TestOpenInvalidAPK(t *testing.T) { reader := strings.NewReader("invalid apk") ctx := context.AddLogger(context.Background()) - handler := apkHandler{} rdr, err := newFileReader(ctx, io.NopCloser(reader)) assert.NoError(t, err) - defer rdr.Close() + defer func() { _ = rdr.Close() }() - archiveChan := make(chan DataOrErr) - - err = handler.processAPK(ctx, rdr, archiveChan) + _, err = createZipReader(rdr) + // Since processAPK no longer creates the zip reader, this now calls createZipReader directly to test the same + // error path. assert.Contains(t, err.Error(), "zip: not a valid zip file") } @@ -91,20 +91,55 @@ func TestOpenValidZipInvalidAPK(t *testing.T) { resp, err := http.Get(validZipURL) assert.NoError(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) - defer resp.Body.Close() - - handler := newAPKHandler() + defer func() { _ = resp.Body.Close() }() newReader, err := newFileReader(context.Background(), resp.Body) if err != nil { t.Errorf("error creating reusable reader: %s", err) } assert.NoError(t, err) - defer newReader.Close() + defer func() { _ = newReader.Close() }() - archiveChan := make(chan DataOrErr) - ctx := context.AddLogger(context.Background()) + zipReader, err := createZipReader(newReader) + assert.NoError(t, err) - err = handler.processAPK(ctx, newReader, archiveChan) + _, err = parseResTable(zipReader) assert.Contains(t, err.Error(), "resources.arsc file not found") } + +type mockResourceEntry struct { + data *resourceEntryData + err error +} + +type mockResourceProvider struct { + entries map[uint32]mockResourceEntry +} + +func (m *mockResourceProvider) GetEntry(id uint32) (*resourceEntryData, error) { + e, ok := m.entries[id] + if !ok { + return nil, nil + } + return e.data, e.err +} + +func TestExtractStringsFromResTable_SkipsBadEntries(t *testing.T) { + provider := &mockResourceProvider{ + entries: map[uint32]mockResourceEntry{ + 0x7f040000: {data: &resourceEntryData{Key: "app_name", ResourceType: "string", Value: "MyApp"}}, + 0x7f040001: {data: &resourceEntryData{Key: "bad_entry", ResourceType: "string"}, err: apkparser.ErrUnknownResourceDataType}, + 0x7f040002: {data: &resourceEntryData{Key: "api_key", ResourceType: "string", Value: "secret123"}}, + 0x7f040003: {data: &resourceEntryData{Key: "icon", ResourceType: "drawable"}}, + }, + } + + rdr := extractStringsFromResTable(provider) + data, err := io.ReadAll(rdr) + assert.NoError(t, err) + + output := string(data) + assert.Contains(t, output, "app_name: MyApp") + assert.Contains(t, output, "api_key: secret123") + assert.NotContains(t, output, "bad_entry") +} diff --git a/pkg/handlers/ar_test.go b/pkg/handlers/ar_test.go index 2f908e4d7c64..64cb694318fe 100644 --- a/pkg/handlers/ar_test.go +++ b/pkg/handlers/ar_test.go @@ -13,14 +13,14 @@ import ( func TestHandleARFile(t *testing.T) { file, err := os.Open("testdata/test.deb") assert.Nil(t, err) - defer file.Close() + defer func() { _ = file.Close() }() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() rdr, err := newFileReader(ctx, file) assert.NoError(t, err) - defer rdr.Close() + defer func() { _ = rdr.Close() }() handler := newARHandler() dataOrErrChan := handler.HandleFile(context.AddLogger(ctx), rdr) diff --git a/pkg/handlers/archive.go b/pkg/handlers/archive.go index c3bf11de7adc..374144ab5053 100644 --- a/pkg/handlers/archive.go +++ b/pkg/handlers/archive.go @@ -140,7 +140,7 @@ func (h *archiveHandler) openArchive( if err != nil { return fmt.Errorf("error opening decompressor with format: %s %w", reader.format.MediaType(), err) } - defer compReader.Close() + defer func() { _ = compReader.Close() }() rdr, err := newFileReader(ctx, compReader) if err != nil { @@ -154,7 +154,7 @@ func (h *archiveHandler) openArchive( err, ) } - defer rdr.Close() + defer func() { _ = rdr.Close() }() return h.openArchive(ctx, depth+1, rdr, dataOrErrChan) case archives.Extractor: @@ -211,7 +211,7 @@ func (h *archiveHandler) extractorHandler(dataOrErrChan chan DataOrErr) func(con if err != nil { return fmt.Errorf("error opening file %s: %w", file.Name(), err) } - defer f.Close() + defer func() { _ = f.Close() }() // Archiver v4 is in alpha and using an experimental version of // rardecode. There is a bug somewhere with rar decoder format 29 @@ -237,7 +237,7 @@ func (h *archiveHandler) extractorHandler(dataOrErrChan chan DataOrErr) func(con } return fmt.Errorf("error creating reader for file %s: %w", file.Name(), err) } - defer rdr.Close() + defer func() { _ = rdr.Close() }() h.metrics.incFilesProcessed() h.metrics.observeFileSize(fileSize) diff --git a/pkg/handlers/archive_test.go b/pkg/handlers/archive_test.go index 7b82b1ebaa8f..9fd5668aaf6b 100644 --- a/pkg/handlers/archive_test.go +++ b/pkg/handlers/archive_test.go @@ -2,9 +2,9 @@ package handlers import ( "context" + regexp "github.com/wasilibs/go-re2" "io" "net/http" - "regexp" "strings" "testing" @@ -81,7 +81,7 @@ func TestArchiveHandler(t *testing.T) { resp, err := http.Get(testCase.archiveURL) assert.NoError(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() handler := newArchiveHandler() @@ -89,7 +89,7 @@ func TestArchiveHandler(t *testing.T) { if err != nil { t.Errorf("error creating reusable reader: %s", err) } - defer newReader.Close() + defer func() { _ = newReader.Close() }() dataOrErrChan := handler.HandleFile(logContext.Background(), newReader) if testCase.expectErr { @@ -121,7 +121,7 @@ func TestOpenInvalidArchive(t *testing.T) { rdr, err := newFileReader(ctx, io.NopCloser(reader)) assert.NoError(t, err) - defer rdr.Close() + defer func() { _ = rdr.Close() }() dataOrErrChan := make(chan DataOrErr) diff --git a/pkg/handlers/default_test.go b/pkg/handlers/default_test.go index 9220e182aa56..37e5dc5c93b0 100644 --- a/pkg/handlers/default_test.go +++ b/pkg/handlers/default_test.go @@ -16,14 +16,14 @@ import ( func TestHandleNonArchiveFile(t *testing.T) { file, err := os.Open("testdata/nonarchive.txt") assert.Nil(t, err) - defer file.Close() + defer func() { _ = file.Close() }() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() rdr, err := newFileReader(ctx, file) assert.NoError(t, err) - defer rdr.Close() + defer func() { _ = rdr.Close() }() handler := newDefaultHandler(defaultHandlerType) dataOrErrChan := handler.HandleFile(context.AddLogger(ctx), rdr) diff --git a/pkg/handlers/handlers.go b/pkg/handlers/handlers.go index 4f98eeef3342..9c6b0c6a6d04 100644 --- a/pkg/handlers/handlers.go +++ b/pkg/handlers/handlers.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "path/filepath" "github.com/gabriel-vasile/mimetype" "github.com/mholt/archives" @@ -54,14 +53,6 @@ var ( ErrProcessingWarning = errors.New("error processing file") ) -type readerConfig struct{ fileExtension string } - -type readerOption func(*readerConfig) - -func withFileExtension(ext string) readerOption { - return func(c *readerConfig) { c.fileExtension = ext } -} - // mimeTypeReader wraps an io.Reader with MIME type information. // This type is used to pass content through the processing pipeline // while carrying its detected MIME type, avoiding redundant type detection. @@ -103,12 +94,7 @@ func newMimeTypeReader(r io.Reader) (mimeTypeReader, error) { // newFileReader creates a fileReader from an io.Reader, optionally using BufferedFileWriter for certain formats. // The caller is responsible for closing the reader when it is no longer needed. -func newFileReader(ctx context.Context, r io.Reader, options ...readerOption) (fReader fileReader, err error) { - var cfg readerConfig - - for _, opt := range options { - opt(&cfg) - } +func newFileReader(ctx context.Context, r io.Reader) (fReader fileReader, err error) { // To detect the MIME type of the input data, we need a reader that supports seeking. // This allows us to read the data multiple times if necessary without losing the original position. // We use a BufferedReaderSeeker to wrap the original reader, enabling this functionality. @@ -136,15 +122,26 @@ func newFileReader(ctx context.Context, r io.Reader, options ...readerOption) (f return fReader, fmt.Errorf("error resetting reader after MIME detection: %w", err) } - // Check for APK files - if shouldHandleAsAPK(cfg, fReader) { - isAPK, err := isAPKFile(&fReader) - if err != nil { - return fReader, fmt.Errorf("error checking for APK: %w", err) - } - if isAPK { + // Detection is content-based, so this now runs for every zip/jar rather than only for files with an apk extension. + if shouldHandleAsAPK(fReader) { + // A failure here (e.g. truncated zip, corrupted central directory, or a polyglot that mimetype still + // reports as zip/jar) only means the file isn't a parseable APK. Since this check now runs for every + // zip/jar rather than only files with an .apk extension, we must not treat the error as fatal; otherwise + // any such file would be skipped entirely instead of falling through to the archive or default handler. + isAPK, apkErr := isAPKFile(&fReader) + switch { + case apkErr != nil: + logContext.AddLogger(ctx).Logger().V(3).Info( + "error checking for APK, falling back to normal handling", "error", apkErr) + case isAPK: return handleAPKFile(&fReader) } + + // isAPKFile inspects the archive via zip.NewReader, which leaves the reader positioned mid-stream. + // Reset to the start so archive identification and any downstream handler read from the beginning. + if _, seekErr := fReader.Seek(0, io.SeekStart); seekErr != nil { + return fReader, fmt.Errorf("error resetting reader after APK check: %w", seekErr) + } } // If a MIME type is known to not be an archive type, we might as well return here rather than @@ -224,7 +221,6 @@ const ( rpmHandlerType handlerType = "rpm" apkHandlerType handlerType = "apk" defaultHandlerType handlerType = "default" - apkExt = ".apk" ) type mimeType string @@ -359,8 +355,7 @@ func HandleFile( return errors.New("reader is nil") } - readerOption := withFileExtension(getFileExtension(chunkSkel)) - rdr, err := newFileReader(ctx, reader, readerOption) + rdr, err := newFileReader(ctx, reader) if err != nil { if errors.Is(err, ErrEmptyReader) { ctx.Logger().V(5).Info("empty reader, skipping file") @@ -425,7 +420,7 @@ func handleChunksWithError( if isFatal(dataOrErr.Err) { return dataOrErr.Err } - ctx.Logger().Error(dataOrErr.Err, "non-critical error processing chunk") + ctx.Logger().V(2).Info("non-critical error processing chunk", "error", dataOrErr.Err) continue } if len(dataOrErr.Data) > 0 { @@ -492,91 +487,14 @@ func isFatal(err error) bool { } } -// getFileExtension extracts the file extension from the chunk's SourceMetadata. -// It considers all sources defined in the MetaData message. -// Note: Probably should add this as a method to the source_metadatapb object. -// then it'd just be chunkSkel.SourceMetadata.GetFileExtension() -func getFileExtension(chunkSkel *sources.Chunk) string { - if chunkSkel == nil || chunkSkel.SourceMetadata == nil { - return "" - } - - var fileName string - - // Inspect the SourceMetadata to determine the source type - switch metadata := chunkSkel.SourceMetadata.Data.(type) { - case *source_metadatapb.MetaData_Artifactory: - fileName = metadata.Artifactory.Path - case *source_metadatapb.MetaData_Azure: - fileName = metadata.Azure.File - case *source_metadatapb.MetaData_AzureRepos: - fileName = metadata.AzureRepos.File - case *source_metadatapb.MetaData_Bitbucket: - fileName = metadata.Bitbucket.File - case *source_metadatapb.MetaData_Buildkite: - fileName = metadata.Buildkite.Link - case *source_metadatapb.MetaData_Circleci: - fileName = metadata.Circleci.Link - case *source_metadatapb.MetaData_Confluence: - fileName = metadata.Confluence.File - case *source_metadatapb.MetaData_Docker: - fileName = metadata.Docker.File - case *source_metadatapb.MetaData_Ecr: - fileName = metadata.Ecr.File - case *source_metadatapb.MetaData_Filesystem: - fileName = metadata.Filesystem.File - case *source_metadatapb.MetaData_Git: - fileName = metadata.Git.File - case *source_metadatapb.MetaData_Github: - fileName = metadata.Github.File - case *source_metadatapb.MetaData_Gitlab: - fileName = metadata.Gitlab.File - case *source_metadatapb.MetaData_Gcs: - fileName = metadata.Gcs.Filename - case *source_metadatapb.MetaData_GoogleDrive: - fileName = metadata.GoogleDrive.File - case *source_metadatapb.MetaData_Huggingface: - fileName = metadata.Huggingface.File - case *source_metadatapb.MetaData_Jira: - fileName = metadata.Jira.Link - case *source_metadatapb.MetaData_Jenkins: - fileName = metadata.Jenkins.Link - case *source_metadatapb.MetaData_Npm: - fileName = metadata.Npm.File - case *source_metadatapb.MetaData_Pypi: - fileName = metadata.Pypi.File - case *source_metadatapb.MetaData_S3: - fileName = metadata.S3.File - case *source_metadatapb.MetaData_Slack: - fileName = metadata.Slack.File - case *source_metadatapb.MetaData_Sharepoint: - fileName = metadata.Sharepoint.Link - case *source_metadatapb.MetaData_Gerrit: - fileName = metadata.Gerrit.File - case *source_metadatapb.MetaData_Test: - fileName = metadata.Test.File - case *source_metadatapb.MetaData_Teams: - fileName = metadata.Teams.File - case *source_metadatapb.MetaData_TravisCI: - fileName = metadata.TravisCI.Link - // Add other sources if they have a file or equivalent field - // Skipping Syslog, Forager, Postman, Vector, Webhook and Elasticsearch - default: - return "" - } - - // Use filepath.Ext to extract the file extension from the file name - ext := filepath.Ext(fileName) - return ext -} - -// shouldHandleAsAPK checks if the file should be handled as an APK based on config and MIME type. -// Note: We can't extend the mimetype package with an APK detection function b/c it would require adjusting settings +// shouldHandleAsAPK reports whether the file should be inspected as a potential APK. +// Detection is content-based (file magic): after confirming the file is a zip/jar, the caller runs isAPKFile, which +// inspects the archive entries for APK markers such as AndroidManifest.xml and classes.dex. +// It deliberately does NOT rely on the file extension so APKs are detected regardless of the source or filename. +// Note: Don't extend the mimetype package with APK detection function. It would require adjusting settings // so that all files are fully read into a byte slice for detection (mimetype.SetLimit(0)), which would bloat memory. -// Instead we call the isAPKFile function in here after ensuring it's a zip/jar file and has an .apk extension. -func shouldHandleAsAPK(cfg readerConfig, fReader fileReader) bool { +func shouldHandleAsAPK(fReader fileReader) bool { return feature.EnableAPKHandler.Load() && - cfg.fileExtension == apkExt && (fReader.mime.String() == string(zipMime) || fReader.mime.String() == string(jarMime)) } diff --git a/pkg/handlers/handlers_test.go b/pkg/handlers/handlers_test.go index 7d8218fc3dc4..db09100672af 100644 --- a/pkg/handlers/handlers_test.go +++ b/pkg/handlers/handlers_test.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" stdctx "context" + _ "embed" "errors" "fmt" "io" @@ -25,6 +26,12 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) +//go:embed testdata/aws-canary-creds.zip +var awsCanaryCredsZip []byte + +//go:embed testdata/sm.zip +var smJSONZip []byte + func TestHandleFileCancelledContext(t *testing.T) { reporter := sources.ChanReporter{Ch: make(chan *sources.Chunk, 2)} @@ -39,26 +46,18 @@ func TestHandleFile(t *testing.T) { reporter := sources.ChanReporter{Ch: make(chan *sources.Chunk, 513)} // Only one chunk is sent on the channel. - // TODO: Embed a zip without making an HTTP request. - resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/aws-canary-creds.zip") - assert.NoError(t, err) - defer func() { - if resp != nil && resp.Body != nil { - resp.Body.Close() - } - }() - assert.Equal(t, 0, len(reporter.Ch)) - assert.NoError(t, HandleFile(context.Background(), resp.Body, &sources.Chunk{}, reporter)) + assert.NoError(t, HandleFile(context.Background(), bytes.NewReader(awsCanaryCredsZip), &sources.Chunk{}, reporter)) assert.Equal(t, 1, len(reporter.Ch)) } +// Fetched over HTTP; the ~5 MB upstream fixture would notably grow pkg/handlers/testdata/ if embedded. func TestHandleHTTPJson(t *testing.T) { - resp, err := http.Get("https://raw.githubusercontent.com/ahrav/nothing-to-see-here/main/sm_random_data.json") + resp, err := http.Get("https://raw.githubusercontent.com/trufflesecurity/trufflehog-test-assets/main/sm_random_data.json") assert.NoError(t, err) defer func() { if resp != nil && resp.Body != nil { - resp.Body.Close() + _ = resp.Body.Close() } }() @@ -78,18 +77,10 @@ func TestHandleHTTPJson(t *testing.T) { } func TestHandleHTTPJsonZip(t *testing.T) { - resp, err := http.Get("https://raw.githubusercontent.com/ahrav/nothing-to-see-here/main/sm.zip") - assert.NoError(t, err) - defer func() { - if resp != nil && resp.Body != nil { - resp.Body.Close() - } - }() - chunkCh := make(chan *sources.Chunk, 1) go func() { defer close(chunkCh) - err := HandleFile(context.Background(), resp.Body, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh}) + err := HandleFile(context.Background(), bytes.NewReader(smJSONZip), &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh}) assert.NoError(t, err) }() @@ -105,21 +96,12 @@ func BenchmarkHandleHTTPJsonZip(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { func() { - resp, err := http.Get("https://raw.githubusercontent.com/ahrav/nothing-to-see-here/main/sm.zip") - assert.NoError(b, err) - - defer func() { - if resp != nil && resp.Body != nil { - resp.Body.Close() - } - }() - chunkCh := make(chan *sources.Chunk, 1) b.StartTimer() go func() { defer close(chunkCh) - err := HandleFile(context.Background(), resp.Body, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh}) + err := HandleFile(context.Background(), bytes.NewReader(smJSONZip), &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh}) assert.NoError(b, err) }() @@ -134,7 +116,7 @@ func BenchmarkHandleHTTPJsonZip(b *testing.B) { func BenchmarkHandleFile(b *testing.B) { file, err := os.Open("testdata/test.tgz") assert.Nil(b, err) - defer file.Close() + defer func() { _ = file.Close() }() b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -323,7 +305,7 @@ func TestHandleFileDOC(t *testing.T) { func BenchmarkHandleAR(b *testing.B) { file, err := os.Open("testdata/test.deb") assert.Nil(b, err) - defer file.Close() + defer func() { _ = file.Close() }() b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -382,7 +364,7 @@ func TestExtractTarContentWithEmptyFile(t *testing.T) { func TestHandleTar(t *testing.T) { file, err := os.Open("testdata/test.tar") assert.Nil(t, err) - defer file.Close() + defer func() { _ = file.Close() }() chunkCh := make(chan *sources.Chunk, 1) go func() { @@ -402,7 +384,7 @@ func TestHandleTar(t *testing.T) { func BenchmarkHandleTar(b *testing.B) { file, err := os.Open("testdata/test.tar") assert.Nil(b, err) - defer file.Close() + defer func() { _ = file.Close() }() b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -424,15 +406,16 @@ func BenchmarkHandleTar(b *testing.B) { } } +// Fetched over HTTP; the ~18 MB upstream fixture would substantially grow pkg/handlers/testdata/ if embedded. func TestHandleLargeHTTPJson(t *testing.T) { - resp, err := http.Get("https://raw.githubusercontent.com/ahrav/nothing-to-see-here/main/md_random_data.json.zip") + resp, err := http.Get("https://raw.githubusercontent.com/trufflesecurity/trufflehog-test-assets/main/md_random_data.json.zip") if !assert.NoError(t, err) { return } defer func() { if resp != nil && resp.Body != nil { - resp.Body.Close() + _ = resp.Body.Close() } }() @@ -455,10 +438,10 @@ func TestHandlePipe(t *testing.T) { r, w := io.Pipe() go func() { - defer w.Close() + defer func() { _ = w.Close() }() file, err := os.Open("testdata/test.tar") assert.NoError(t, err) - defer file.Close() + defer func() { _ = file.Close() }() _, err = io.Copy(w, file) assert.NoError(t, err) }() @@ -543,7 +526,7 @@ func TestHandleGitCatFile(t *testing.T) { } else { gitDir = setupTempGitRepoWithUnsupportedFile(t, tt.fileName, tt.fileSize) } - defer os.RemoveAll(gitDir) + defer func() { _ = os.RemoveAll(gitDir) }() cmd := exec.Command("git", "-C", gitDir, "rev-parse", "HEAD") hashBytes, err := cmd.Output() @@ -634,7 +617,7 @@ func setupTempGitRepoCommon(t *testing.T, fileName string, fileSize int, isUnsup if err != nil { t.Fatalf("Failed to create file: %v", err) } - defer f.Close() + defer func() { _ = f.Close() }() if isUnsupported { // Write ELF header for unsupported file. @@ -758,7 +741,7 @@ func TestHandleGitCatFileWithPipeError(t *testing.T) { simulatedError := errors.New("simulated error during newFileReader") gitDir := setupTempGitRepo(t, fileName, fileSize) - defer os.RemoveAll(gitDir) + defer func() { _ = os.RemoveAll(gitDir) }() commitHash := getGitCommitHash(t, gitDir) diff --git a/pkg/handlers/rpm_test.go b/pkg/handlers/rpm_test.go index ba4d5d622369..a48d8d83daac 100644 --- a/pkg/handlers/rpm_test.go +++ b/pkg/handlers/rpm_test.go @@ -13,14 +13,14 @@ import ( func TestHandleRPMFile(t *testing.T) { file, err := os.Open("testdata/test.rpm") assert.Nil(t, err) - defer file.Close() + defer func() { _ = file.Close() }() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() rdr, err := newFileReader(ctx, file) assert.NoError(t, err) - defer rdr.Close() + defer func() { _ = rdr.Close() }() handler := newRPMHandler() dataOrErrChan := handler.HandleFile(context.AddLogger(ctx), rdr) diff --git a/pkg/handlers/testdata/aws-canary-creds.zip b/pkg/handlers/testdata/aws-canary-creds.zip new file mode 100644 index 000000000000..8c803c340252 Binary files /dev/null and b/pkg/handlers/testdata/aws-canary-creds.zip differ diff --git a/pkg/handlers/testdata/sm.zip b/pkg/handlers/testdata/sm.zip new file mode 100644 index 000000000000..7dfc4552fe50 Binary files /dev/null and b/pkg/handlers/testdata/sm.zip differ diff --git a/pkg/iobuf/bufferedreaderseeker.go b/pkg/iobuf/bufferedreaderseeker.go index b42ff72b8d1b..4fbfbd886cf9 100644 --- a/pkg/iobuf/bufferedreaderseeker.go +++ b/pkg/iobuf/bufferedreaderseeker.go @@ -390,7 +390,7 @@ func (br *BufferedReadSeeker) Close() error { } if br.tempFile != nil { - br.tempFile.Close() + _ = br.tempFile.Close() return os.Remove(br.tempFileName) } return nil diff --git a/pkg/output/github_actions.go b/pkg/output/github_actions.go index 878a2de725d3..e2197eff9de1 100644 --- a/pkg/output/github_actions.go +++ b/pkg/output/github_actions.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "strings" "sync" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -18,10 +19,10 @@ type GitHubActionsPrinter struct{ mu sync.Mutex } func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error { out := gitHubActionsOutputFormat{ - DetectorType: r.Result.DetectorType.String(), + DetectorType: r.DetectorType.String(), DetectorDescription: r.DetectorDescription, DecoderType: r.DecoderType.String(), - Verified: r.Result.Verified, + Verified: r.Verified, } meta, err := structToMap(r.SourceMetadata.Data) @@ -29,20 +30,7 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM return fmt.Errorf("could not marshal result: %w", err) } - for _, data := range meta { - for k, v := range data { - if k == "line" { - if line, ok := v.(float64); ok { - out.StartLine = int64(line) - } - } - if k == "file" { - if filename, ok := v.(string); ok { - out.Filename = filename - } - } - } - } + out.Filename, out.StartLine = extractFileAndLine(meta) verifiedStatus := "unverified" if out.Verified { @@ -61,7 +49,7 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM dedupeCache[key] = struct{}{} name := "" - if nameValue, ok := r.Result.ExtraData["name"]; ok { + if nameValue, ok := r.ExtraData["name"]; ok { name = fmt.Sprintf(" (%s)", nameValue) } @@ -70,12 +58,30 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM message = fmt.Sprintf("Found %s %s%s result with %s encoding 🐷🔑\n", verifiedStatus, out.DetectorType, name, out.DecoderType) } - fmt.Printf("::warning file=%s,line=%d,endLine=%d::%s", - out.Filename, out.StartLine, out.StartLine, message) + fmt.Print(formatWarningCommand(out.Filename, out.StartLine, message)) return nil } +func formatWarningCommand(file string, line int64, message string) string { + return fmt.Sprintf("::warning file=%s,line=%d,endLine=%d::%s\n", + escapeWorkflowProperty(file), line, line, escapeWorkflowData(strings.TrimSuffix(message, "\n"))) +} + +func escapeWorkflowData(s string) string { + s = strings.ReplaceAll(s, "%", "%25") + s = strings.ReplaceAll(s, "\r", "%0D") + s = strings.ReplaceAll(s, "\n", "%0A") + return s +} + +func escapeWorkflowProperty(s string) string { + s = escapeWorkflowData(s) + s = strings.ReplaceAll(s, ":", "%3A") + s = strings.ReplaceAll(s, ",", "%2C") + return s +} + type gitHubActionsOutputFormat struct { DetectorType string DetectorDescription string diff --git a/pkg/output/github_actions_test.go b/pkg/output/github_actions_test.go new file mode 100644 index 000000000000..770af166936f --- /dev/null +++ b/pkg/output/github_actions_test.go @@ -0,0 +1,97 @@ +package output + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" +) + +// gitResultWithFile builds a verified git-family result whose file path is caller-controlled, +// mirroring a path decoded by pkg/gitparse that can contain characters git allows in paths, +// such as a newline. +func gitResultWithFile(file string) *detectors.ResultWithMetadata { + return &detectors.ResultWithMetadata{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{File: file, Line: 1}, + }, + }, + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + SourceName: "my-repo", + Result: detectors.Result{ + DetectorType: detector_typepb.DetectorType_Github, + Raw: []byte("secret"), + Verified: false, + }, + } +} + +func TestEscapeWorkflowData(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"plain text is untouched", "config/prod.yaml", "config/prod.yaml"}, + {"newline is escaped", "a\nb", "a%0Ab"}, + {"carriage return is escaped", "a\rb", "a%0Db"}, + {"percent is escaped first so it can't unmask other sequences", "100%\n", "100%25%0A"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, escapeWorkflowData(tt.in)) + }) + } +} + +func TestEscapeWorkflowProperty(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"plain text is untouched", "config/prod.yaml", "config/prod.yaml"}, + {"colon is escaped", "a:b", "a%3Ab"}, + {"comma is escaped", "a,b", "a%2Cb"}, + {"newline is escaped, same as data", "a\nb", "a%0Ab"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, escapeWorkflowProperty(tt.in)) + }) + } +} + +// TestFormatWarningCommand_EscapesSpecialCharactersInFilename covers a file path containing +// characters git allows but the workflow-command syntax does not: git permits newlines in file +// paths, and gitparse decodes the git-quoted diff-header form back into a real newline (see +// pkg/gitparse pathFromToFileLine). Unescaped, an embedded newline followed by "::" would start +// what looks like a second workflow command on its own line. +func TestFormatWarningCommand_EscapesSpecialCharactersInFilename(t *testing.T) { + unusualFile := "readme.md\n::error title=example::some text\n::add-mask::some-value" + + got := formatWarningCommand(unusualFile, 1, "Found unverified result\n") + + require.Equal(t, 1, strings.Count(got, "\n"), "output must be exactly one line (plus its trailing newline)") + assert.NotContains(t, got, "\n::error", "no unescaped newline should start a new line") + assert.NotContains(t, got, "\n::add-mask", "no unescaped newline should start a new line") + assert.Contains(t, got, "%0A", "the embedded newlines must survive, escaped, rather than being silently dropped") + assert.True(t, strings.HasPrefix(got, "::warning file="), "the tool's own command must still be well-formed") +} + +func TestGitHubActionsPrinter_Print_HandlesFilenameWithSpecialCharacters(t *testing.T) { + dedupeCache = make(map[string]struct{}) // isolate from other tests sharing the package-level cache + + p := &GitHubActionsPrinter{} + unusualFile := "readme.md\n::stop-commands::deadbeef" + + require.NoError(t, p.Print(context.Background(), gitResultWithFile(unusualFile))) +} diff --git a/pkg/output/json.go b/pkg/output/json.go index 4d8f7eb711f0..f6758cc8dc4e 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -55,6 +55,7 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) Redacted string ExtraData map[string]string StructuredData *detectorspb.StructuredData + SecretParts map[string]string }{ SourceMetadata: r.SourceMetadata, SourceID: r.SourceID, @@ -72,6 +73,7 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) Redacted: r.Redacted, ExtraData: r.ExtraData, StructuredData: r.StructuredData, + SecretParts: r.SecretParts, } out, err := json.Marshal(v) if err != nil { diff --git a/pkg/output/legacy_json.go b/pkg/output/legacy_json.go index 74683f4573f9..5f3cf3ff6ea1 100644 --- a/pkg/output/legacy_json.go +++ b/pkg/output/legacy_json.go @@ -85,7 +85,7 @@ func convertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*Leg diff := GenerateDiff(commit, fileName) - foundString := string(r.Result.Raw) + foundString := string(r.Raw) // Add highlighting to the offending bit of string. printableDiff := strings.ReplaceAll(diff, foundString, fmt.Sprintf("\u001b[93m%s\u001b[0m", foundString)) @@ -99,7 +99,7 @@ func convertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*Leg Diff: diff, Path: fileName, PrintDiff: printableDiff, - Reason: r.Result.DetectorType.String(), + Reason: r.DetectorType.String(), StringsFound: []string{foundString}, } return output, nil diff --git a/pkg/output/plain.go b/pkg/output/plain.go index a1df13de7bce..14f972120a13 100644 --- a/pkg/output/plain.go +++ b/pkg/output/plain.go @@ -30,16 +30,16 @@ type PlainPrinter struct{ mu sync.Mutex } func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error { out := outputFormat{ - DetectorType: r.Result.DetectorType.String(), + DetectorType: r.DetectorType.String(), DecoderType: r.DecoderType.String(), - Verified: r.Result.Verified, - VerificationError: r.Result.VerificationError(), + Verified: r.Verified, + VerificationError: r.VerificationError(), MetaData: r.SourceMetadata, - Raw: strings.TrimSpace(string(r.Result.Raw)), + Raw: strings.TrimSpace(string(r.Raw)), DetectorDescription: r.DetectorDescription, } - meta, err := structToMap(out.MetaData.Data) + meta, err := structToMap(out.Data) if err != nil { return fmt.Errorf("could not marshal result: %w", err) } @@ -49,43 +49,42 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) defer p.mu.Unlock() if out.Verified { - boldGreenPrinter.Print("✅ Found verified result 🐷🔑\n") + _, _ = boldGreenPrinter.Print("✅ Found verified result 🐷🔑\n") } else { printer = whitePrinter - boldWhitePrinter.Print("Found unverified result 🐷🔑❓\n") + _, _ = boldWhitePrinter.Print("Found unverified result 🐷🔑❓\n") if out.VerificationError != nil { - yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError) + _, _ = yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError) } } if r.VerificationFromCache { - cyanPrinter.Print("(🔍 Using cached verification)\n") + _, _ = cyanPrinter.Print("(🔍 Using cached verification)\n") } + _, _ = printer.Printf("Detector Type: %s\n", out.DetectorType) + _, _ = printer.Printf("Decoder Type: %s\n", out.DecoderType) + _, _ = printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw)) - printer.Printf("Detector Type: %s\n", out.DetectorType) - printer.Printf("Decoder Type: %s\n", out.DecoderType) - printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw)) - - for k, v := range r.Result.ExtraData { - printer.Printf( + for k, v := range r.ExtraData { + _, _ = printer.Printf( "%s: %v\n", cases.Title(language.AmericanEnglish).String(k), v) } - if r.Result.StructuredData != nil { - for idx, v := range r.Result.StructuredData.GithubSshKey { - printer.Printf("GithubSshKey %d User: %s\n", idx, v.User) + if r.StructuredData != nil { + for idx, v := range r.StructuredData.GithubSshKey { + _, _ = printer.Printf("GithubSshKey %d User: %s\n", idx, v.User) if v.PublicKeyFingerprint != "" { - printer.Printf("GithubSshKey %d Fingerprint: %s\n", idx, v.PublicKeyFingerprint) + _, _ = printer.Printf("GithubSshKey %d Fingerprint: %s\n", idx, v.PublicKeyFingerprint) } } - for idx, v := range r.Result.StructuredData.TlsPrivateKey { - printer.Printf("TlsPrivateKey %d Fingerprint: %s\n", idx, v.CertificateFingerprint) - printer.Printf("TlsPrivateKey %d Verification URL: %s\n", idx, v.VerificationUrl) - printer.Printf("TlsPrivateKey %d Expiration: %d\n", idx, v.ExpirationTimestamp) + for idx, v := range r.StructuredData.TlsPrivateKey { + _, _ = printer.Printf("TlsPrivateKey %d Fingerprint: %s\n", idx, v.CertificateFingerprint) + _, _ = printer.Printf("TlsPrivateKey %d Verification URL: %s\n", idx, v.VerificationUrl) + _, _ = printer.Printf("TlsPrivateKey %d Expiration: %d\n", idx, v.ExpirationTimestamp) } } @@ -100,12 +99,12 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) } sort.Strings(aggregateDataKeys) for _, k := range aggregateDataKeys { - printer.Printf("%s: %v\n", cases.Title(language.AmericanEnglish).String(k), aggregateData[k]) + _, _ = printer.Printf("%s: %v\n", cases.Title(language.AmericanEnglish).String(k), aggregateData[k]) } // if analysis info is not nil, means the detector added key for analyzer and result is verified - if r.Result.SecretParts != nil && r.Result.Verified { - printer.Printf("Analyze: Run `trufflehog analyze` to analyze this key's permissions\n") + if r.SecretParts != nil && r.Verified { + _, _ = printer.Printf("Analyze: Run `trufflehog analyze` to analyze this key's permissions\n") } fmt.Println("") diff --git a/pkg/output/sarif.go b/pkg/output/sarif.go new file mode 100644 index 000000000000..e15ab14e6a5c --- /dev/null +++ b/pkg/output/sarif.go @@ -0,0 +1,268 @@ +package output + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "sync" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/version" +) + +// SARIF (Static Analysis Results Interchange Format) 2.1.0 identifiers. +// See https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html +const ( + sarifSchemaURI = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json" + sarifVersion = "2.1.0" + + // sarifLevelError/sarifLevelWarning map to SARIF's result.level. Verified secrets are + // confirmed live credentials and are reported as errors; unverified secrets matched a + // detector pattern but could not be confirmed, and are reported as warnings so that tools + // consuming SARIF (e.g. GitHub code scanning) can triage them separately. + sarifLevelError = "error" + sarifLevelWarning = "warning" +) + +// SarifPrinter is a printer that accumulates results and, once the scan completes, emits them +// as a single SARIF 2.1.0 log. Unlike the other Printer implementations, SARIF results cannot be +// streamed one-per-line: the spec requires one JSON document containing every run and result, so +// Print only buffers results and Flush performs the actual marshal/write. +// +// TODO: results are held in memory for the full scan and Flush marshals them in one pass, so peak +// memory grows with result count (roughly 2x at Flush, for the struct plus its marshaled JSON). +// Fine for typical scan sizes; if scans with very large result counts start OOMing, switch to +// writing results to the underlying writer incrementally as they're produced in Print. +type SarifPrinter struct { + mu sync.Mutex + results []sarifResult + rules map[string]*sarifRule // keyed by detector type name, de-duplicated across results +} + +// Print buffers a single result for inclusion in the SARIF document written by Flush. +func (p *SarifPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error { + meta, err := structToMap(r.SourceMetadata.Data) + if err != nil { + return fmt.Errorf("could not marshal result: %w", err) + } + + file, line := extractFileAndLine(meta) + ruleID := r.DetectorType.String() + + level := sarifLevelWarning + if r.Verified { + level = sarifLevelError + } + + verifiedStatus := "unverified" + if r.Verified { + verifiedStatus = "verified" + } + + location := sarifLocation{ + PhysicalLocation: sarifPhysicalLocation{ + ArtifactLocation: sarifArtifactLocation{ + URI: sarifArtifactURI(file, r.SourceType.String(), r.SourceName), + }, + }, + } + // SARIF's region is optional; only sources whose metadata carries a line number (git, + // filesystem, S3, etc.) can populate it. Sources like Postman or Elasticsearch have no + // concept of a line, so region is omitted rather than reported as a misleading zero. + if line > 0 { + location.PhysicalLocation.Region = &sarifRegion{StartLine: line} + } + + result := sarifResult{ + RuleID: ruleID, + Level: level, + Message: sarifMessage{Text: fmt.Sprintf("Found %s result for detector %s.", verifiedStatus, ruleID)}, + Locations: []sarifLocation{ + location, + }, + // PartialFingerprints lets GitHub code scanning (and other SARIF consumers) match the + // same finding across scans, so it can track a secret as "new" or "fixed" instead of + // reporting it fresh on every run. + PartialFingerprints: map[string]string{ + "trufflehogFingerprint/v1": sarifFingerprint(ruleID, location.PhysicalLocation.ArtifactLocation.URI, line, r.Raw), + }, + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.rules == nil { + p.rules = make(map[string]*sarifRule) + } + if _, ok := p.rules[ruleID]; !ok { + p.rules[ruleID] = &sarifRule{ + ID: ruleID, + Name: ruleID, + ShortDescription: sarifMessage{Text: r.DetectorDescription}, + } + } + p.results = append(p.results, result) + + return nil +} + +// Flush marshals every result buffered by Print into a single SARIF 2.1.0 log and writes it to +// w. It must be called exactly once, after all Print calls have completed (i.e. once the scan +// has finished), since SARIF is a single JSON document rather than a streamable format. +func (p *SarifPrinter) Flush(w io.Writer) error { + p.mu.Lock() + defer p.mu.Unlock() + + rules := make([]*sarifRule, 0, len(p.rules)) + for _, rule := range p.rules { + rules = append(rules, rule) + } + // Sort for deterministic output; map iteration order is randomized in Go. + sort.Slice(rules, func(i, j int) bool { return rules[i].ID < rules[j].ID }) + + results := p.results + if results == nil { + // Emit an empty array rather than JSON null when nothing was found. + results = []sarifResult{} + } + + doc := sarifLog{ + Schema: sarifSchemaURI, + Version: sarifVersion, + Runs: []sarifRun{ + { + Tool: sarifTool{ + Driver: sarifDriver{ + Name: "trufflehog", + InformationURI: "https://github.com/trufflesecurity/trufflehog", + Version: version.BuildVersion, + Rules: rules, + }, + }, + Results: results, + }, + }, + } + + out, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return fmt.Errorf("could not marshal SARIF output: %w", err) + } + if _, err := w.Write(out); err != nil { + return fmt.Errorf("could not write SARIF output: %w", err) + } + _, err = w.Write([]byte("\n")) + return err +} + +// extractFileAndLine pulls the "file" and "line" fields out of a result's source metadata, if +// present. Most source metadata types (git, filesystem, S3, docker, ...) carry these fields, but +// not all do, so both return values may be zero. +func extractFileAndLine(meta map[string]map[string]any) (file string, line int64) { + for _, data := range meta { + for k, v := range data { + switch k { + case "file": + if f, ok := v.(string); ok { + file = f + } + case "line": + if l, ok := v.(float64); ok { + line = int64(l) + } + } + } + } + return file, line +} + +// sarifArtifactURI returns the best-effort identifier for where a secret was found. It prefers +// the file path from source metadata; when a source has no file concept (e.g. Postman, +// Elasticsearch) it falls back to a "://" URI so the location field is +// never empty, which the SARIF spec requires. +func sarifArtifactURI(file, sourceType, sourceName string) string { + if file != "" { + return file + } + return fmt.Sprintf("%s://%s", strings.ToLower(sourceType), sourceName) +} + +// sarifFingerprint derives a stable identifier for a finding so SARIF consumers can recognize +// the same secret across repeated scans (e.g. to mark it "fixed" once it no longer appears). +// Verification status is deliberately excluded: it's already carried in the result's "level" +// field, and including it here would change the fingerprint (and reset alert history) whenever +// a secret's verification flips between runs. The raw secret value is included so that sources +// with no file/line concept (Postman, Elasticsearch, ...) don't collapse every finding of the +// same detector type into one fingerprint. +func sarifFingerprint(ruleID, uri string, line int64, raw []byte) string { + key := fmt.Sprintf("%s:%s:%d:%x", ruleID, uri, line, sha256.Sum256(raw)) + sum := sha256.Sum256([]byte(key)) + return hex.EncodeToString(sum[:]) +} + +// The following types implement a minimal subset of the SARIF 2.1.0 object model needed to +// describe trufflehog's results. Only fields trufflehog actually populates are included; the +// full spec has many optional fields that aren't relevant here. +type sarifLog struct { + Schema string `json:"$schema"` + Version string `json:"version"` + Runs []sarifRun `json:"runs"` +} + +type sarifRun struct { + Tool sarifTool `json:"tool"` + Results []sarifResult `json:"results"` +} + +type sarifTool struct { + Driver sarifDriver `json:"driver"` +} + +type sarifDriver struct { + Name string `json:"name"` + InformationURI string `json:"informationUri"` + Version string `json:"version"` + Rules []*sarifRule `json:"rules"` +} + +// sarifRule describes a detector as a SARIF "rule". One rule is emitted per distinct detector +// type that produced at least one result. +type sarifRule struct { + ID string `json:"id"` + Name string `json:"name"` + ShortDescription sarifMessage `json:"shortDescription"` +} + +type sarifResult struct { + RuleID string `json:"ruleId"` + Level string `json:"level"` + Message sarifMessage `json:"message"` + Locations []sarifLocation `json:"locations"` + PartialFingerprints map[string]string `json:"partialFingerprints,omitempty"` +} + +type sarifMessage struct { + Text string `json:"text"` +} + +type sarifLocation struct { + PhysicalLocation sarifPhysicalLocation `json:"physicalLocation"` +} + +type sarifPhysicalLocation struct { + ArtifactLocation sarifArtifactLocation `json:"artifactLocation"` + Region *sarifRegion `json:"region,omitempty"` +} + +type sarifArtifactLocation struct { + URI string `json:"uri"` +} + +type sarifRegion struct { + StartLine int64 `json:"startLine"` +} diff --git a/pkg/output/sarif_test.go b/pkg/output/sarif_test.go new file mode 100644 index 000000000000..1c101bcbafa7 --- /dev/null +++ b/pkg/output/sarif_test.go @@ -0,0 +1,146 @@ +package output + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" +) + +// gitResult builds a verified result from a git-family source, which has file/line metadata. +func gitResult(verified bool) *detectors.ResultWithMetadata { + return &detectors.ResultWithMetadata{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{File: "config/prod.yaml", Line: 42}, + }, + }, + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + SourceName: "my-repo", + DetectorDescription: "AWS credentials", + Result: detectors.Result{ + DetectorType: detector_typepb.DetectorType_AWS, + Raw: []byte("secret"), + Verified: verified, + }, + } +} + +// elasticsearchResult builds a result from a source with no file/line concept. +func elasticsearchResult() *detectors.ResultWithMetadata { + return &detectors.ResultWithMetadata{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Elasticsearch{ + Elasticsearch: &source_metadatapb.Elasticsearch{Index: "logs", DocumentId: "123"}, + }, + }, + SourceType: sourcespb.SourceType_SOURCE_TYPE_ELASTICSEARCH, + SourceName: "my-cluster", + DetectorDescription: "Slack token", + Result: detectors.Result{ + DetectorType: detector_typepb.DetectorType_Slack, + Raw: []byte("secret"), + Verified: false, + }, + } +} + +func TestSarifPrinter_PrintAndFlush(t *testing.T) { + p := &SarifPrinter{} + ctx := context.Background() + + require.NoError(t, p.Print(ctx, gitResult(true))) + require.NoError(t, p.Print(ctx, elasticsearchResult())) + + var buf bytes.Buffer + require.NoError(t, p.Flush(&buf)) + + var doc sarifLog + require.NoError(t, json.Unmarshal(buf.Bytes(), &doc)) + + assert.Equal(t, sarifVersion, doc.Version) + assert.Equal(t, sarifSchemaURI, doc.Schema) + require.Len(t, doc.Runs, 1) + + run := doc.Runs[0] + assert.Equal(t, "trufflehog", run.Tool.Driver.Name) + require.Len(t, run.Tool.Driver.Rules, 2, "one rule per distinct detector type") + require.Len(t, run.Results, 2) + + // Verified git result: file/line populated, level "error". + gitFinding := run.Results[0] + assert.Equal(t, "AWS", gitFinding.RuleID) + assert.Equal(t, sarifLevelError, gitFinding.Level) + assert.Equal(t, "config/prod.yaml", gitFinding.Locations[0].PhysicalLocation.ArtifactLocation.URI) + require.NotNil(t, gitFinding.Locations[0].PhysicalLocation.Region) + assert.EqualValues(t, 42, gitFinding.Locations[0].PhysicalLocation.Region.StartLine) + assert.NotEmpty(t, gitFinding.PartialFingerprints["trufflehogFingerprint/v1"]) + + // Unverified elasticsearch result: no file/line, falls back to a source URI, level "warning". + esFinding := run.Results[1] + assert.Equal(t, "Slack", esFinding.RuleID) + assert.Equal(t, sarifLevelWarning, esFinding.Level) + assert.Equal(t, "source_type_elasticsearch://my-cluster", esFinding.Locations[0].PhysicalLocation.ArtifactLocation.URI) + assert.Nil(t, esFinding.Locations[0].PhysicalLocation.Region) +} + +func TestSarifPrinter_FlushWithNoResults(t *testing.T) { + p := &SarifPrinter{} + + var buf bytes.Buffer + require.NoError(t, p.Flush(&buf)) + + var doc sarifLog + require.NoError(t, json.Unmarshal(buf.Bytes(), &doc)) + + require.Len(t, doc.Runs, 1) + assert.NotNil(t, doc.Runs[0].Results, "results must be an empty array, not null, per the SARIF spec") + assert.Empty(t, doc.Runs[0].Results) +} + +func TestSarifFingerprint_StableAndDistinct(t *testing.T) { + a := sarifFingerprint("AWS", "config/prod.yaml", 42, []byte("secret1")) + b := sarifFingerprint("AWS", "config/prod.yaml", 42, []byte("secret1")) + assert.Equal(t, a, b, "fingerprint must be stable across calls with identical inputs") + + c := sarifFingerprint("AWS", "config/prod.yaml", 43, []byte("secret1")) + assert.NotEqual(t, a, c, "fingerprint must change when the finding location changes") + + // Same rule/location but no file/line (e.g. Postman, Elasticsearch) must not collapse + // distinct secrets into the same fingerprint. + d := sarifFingerprint("Slack", "elasticsearch://my-cluster", 0, []byte("secret1")) + e := sarifFingerprint("Slack", "elasticsearch://my-cluster", 0, []byte("secret2")) + assert.NotEqual(t, d, e, "fingerprint must differ for distinct secrets sharing a fileless location") +} + +// TestSarifPrinter_FingerprintStableAcrossVerificationChange guards against regressing to a +// fingerprint that includes verification status: if a secret's status flips between scans (API +// error, rate limit, credential rotation), the fingerprint must stay the same so GitHub code +// scanning tracks it as the same finding rather than closing/reopening an alert. +func TestSarifPrinter_FingerprintStableAcrossVerificationChange(t *testing.T) { + ctx := context.Background() + + unverified := &SarifPrinter{} + require.NoError(t, unverified.Print(ctx, gitResult(false))) + + verified := &SarifPrinter{} + require.NoError(t, verified.Print(ctx, gitResult(true))) + + assert.Equal(t, + unverified.results[0].PartialFingerprints["trufflehogFingerprint/v1"], + verified.results[0].PartialFingerprints["trufflehogFingerprint/v1"], + ) +} + +func TestSarifArtifactURI(t *testing.T) { + assert.Equal(t, "config/prod.yaml", sarifArtifactURI("config/prod.yaml", "SOURCE_TYPE_GIT", "my-repo")) + assert.Equal(t, "source_type_elasticsearch://my-cluster", sarifArtifactURI("", "SOURCE_TYPE_ELASTICSEARCH", "my-cluster")) +} diff --git a/pkg/pb/credentialspb/credentials.pb.go b/pkg/pb/credentialspb/credentials.pb.go index 324edf8349ee..2e1da0b21659 100644 --- a/pkg/pb/credentialspb/credentials.pb.go +++ b/pkg/pb/credentialspb/credentials.pb.go @@ -899,6 +899,69 @@ func (x *GoogleDriveDWD) GetExcludeUsers() []string { return nil } +type BitbucketApiToken struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Email string `protobuf:"bytes,1,opt,name=email,proto3" json:"email,omitempty"` + Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"` + Username string `protobuf:"bytes,3,opt,name=username,proto3" json:"username,omitempty"` // optional; falls back to Bitbucket's documented alias for git ops when omitted +} + +func (x *BitbucketApiToken) Reset() { + *x = BitbucketApiToken{} + if protoimpl.UnsafeEnabled { + mi := &file_credentials_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BitbucketApiToken) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BitbucketApiToken) ProtoMessage() {} + +func (x *BitbucketApiToken) ProtoReflect() protoreflect.Message { + mi := &file_credentials_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BitbucketApiToken.ProtoReflect.Descriptor instead. +func (*BitbucketApiToken) Descriptor() ([]byte, []int) { + return file_credentials_proto_rawDescGZIP(), []int{15} +} + +func (x *BitbucketApiToken) GetEmail() string { + if x != nil { + return x.Email + } + return "" +} + +func (x *BitbucketApiToken) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + +func (x *BitbucketApiToken) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + var File_credentials_proto protoreflect.FileDescriptor var file_credentials_proto_rawDesc = []byte{ @@ -994,11 +1057,17 @@ var file_credentials_proto_rawDesc = []byte{ 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x55, 0x73, 0x65, 0x72, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x55, 0x73, - 0x65, 0x72, 0x73, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, - 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x72, 0x73, 0x22, 0x5b, 0x0a, 0x11, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x41, 0x70, 0x69, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x14, + 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, + 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, + 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, + 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, + 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1013,7 +1082,7 @@ func file_credentials_proto_rawDescGZIP() []byte { return file_credentials_proto_rawDescData } -var file_credentials_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_credentials_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_credentials_proto_goTypes = []interface{}{ (*Unauthenticated)(nil), // 0: credentials.Unauthenticated (*SSHAuth)(nil), // 1: credentials.SSHAuth @@ -1030,6 +1099,7 @@ var file_credentials_proto_goTypes = []interface{}{ (*GitHubApp)(nil), // 12: credentials.GitHubApp (*SlackTokens)(nil), // 13: credentials.SlackTokens (*GoogleDriveDWD)(nil), // 14: credentials.GoogleDriveDWD + (*BitbucketApiToken)(nil), // 15: credentials.BitbucketApiToken } var file_credentials_proto_depIdxs = []int32{ 10, // 0: credentials.SES.creds:type_name -> credentials.AWS @@ -1226,6 +1296,18 @@ func file_credentials_proto_init() { return nil } } + file_credentials_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BitbucketApiToken); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -1233,7 +1315,7 @@ func file_credentials_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_credentials_proto_rawDesc, NumEnums: 0, - NumMessages: 15, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/pb/credentialspb/credentials.pb.validate.go b/pkg/pb/credentialspb/credentials.pb.validate.go index ded5a26b202f..be220da3fcfe 100644 --- a/pkg/pb/credentialspb/credentials.pb.validate.go +++ b/pkg/pb/credentialspb/credentials.pb.validate.go @@ -1666,3 +1666,111 @@ var _ interface { Cause() error ErrorName() string } = GoogleDriveDWDValidationError{} + +// Validate checks the field values on BitbucketApiToken with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *BitbucketApiToken) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on BitbucketApiToken with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// BitbucketApiTokenMultiError, or nil if none found. +func (m *BitbucketApiToken) ValidateAll() error { + return m.validate(true) +} + +func (m *BitbucketApiToken) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Email + + // no validation rules for Token + + // no validation rules for Username + + if len(errors) > 0 { + return BitbucketApiTokenMultiError(errors) + } + + return nil +} + +// BitbucketApiTokenMultiError is an error wrapping multiple validation errors +// returned by BitbucketApiToken.ValidateAll() if the designated constraints +// aren't met. +type BitbucketApiTokenMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m BitbucketApiTokenMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m BitbucketApiTokenMultiError) AllErrors() []error { return m } + +// BitbucketApiTokenValidationError is the validation error returned by +// BitbucketApiToken.Validate if the designated constraints aren't met. +type BitbucketApiTokenValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e BitbucketApiTokenValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e BitbucketApiTokenValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e BitbucketApiTokenValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e BitbucketApiTokenValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e BitbucketApiTokenValidationError) ErrorName() string { + return "BitbucketApiTokenValidationError" +} + +// Error satisfies the builtin error interface +func (e BitbucketApiTokenValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sBitbucketApiToken.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = BitbucketApiTokenValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = BitbucketApiTokenValidationError{} diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.go index 284bf8546ada..7341903a80db 100644 --- a/pkg/pb/custom_detectorspb/custom_detectors.pb.go +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.go @@ -204,6 +204,7 @@ type VerifierConfig struct { Unsafe bool `protobuf:"varint,2,opt,name=unsafe,proto3" json:"unsafe,omitempty"` Headers []string `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty"` SuccessRanges []string `protobuf:"bytes,4,rep,name=successRanges,proto3" json:"successRanges,omitempty"` + RotatedRanges []string `protobuf:"bytes,5,rep,name=rotatedRanges,proto3" json:"rotatedRanges,omitempty"` } func (x *VerifierConfig) Reset() { @@ -266,6 +267,13 @@ func (x *VerifierConfig) GetSuccessRanges() []string { return nil } +func (x *VerifierConfig) GetRotatedRanges() []string { + if x != nil { + return x.RotatedRanges + } + return nil +} + type ValidationConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -391,7 +399,7 @@ var file_custom_detectors_proto_rawDesc = []byte{ 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb4, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, @@ -400,25 +408,27 @@ var file_custom_detectors_proto_rawDesc = []byte{ 0x64, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0xcb, 0x01, 0x0a, 0x10, 0x56, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x25, - 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x5f, 0x64, 0x69, 0x67, 0x69, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, - 0x44, 0x69, 0x67, 0x69, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x73, 0x5f, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x63, 0x61, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x4c, 0x6f, 0x77, 0x65, 0x72, - 0x63, 0x61, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, - 0x5f, 0x75, 0x70, 0x70, 0x65, 0x72, 0x63, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x55, 0x70, 0x70, 0x65, 0x72, 0x63, - 0x61, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x5f, - 0x73, 0x70, 0x65, 0x63, 0x69, 0x61, 0x6c, 0x5f, 0x63, 0x68, 0x61, 0x72, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x13, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x53, 0x70, 0x65, 0x63, - 0x69, 0x61, 0x6c, 0x43, 0x68, 0x61, 0x72, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, - 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, - 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, - 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x72, 0x6f, 0x74, + 0x61, 0x74, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0d, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, + 0xcb, 0x01, 0x0a, 0x10, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, + 0x5f, 0x64, 0x69, 0x67, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x44, 0x69, 0x67, 0x69, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x5f, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x63, 0x61, 0x73, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x73, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x63, 0x61, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, 0x5f, 0x75, 0x70, 0x70, 0x65, 0x72, 0x63, 0x61, 0x73, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x73, + 0x55, 0x70, 0x70, 0x65, 0x72, 0x63, 0x61, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x73, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x69, 0x61, 0x6c, 0x5f, 0x63, 0x68, + 0x61, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x73, 0x53, 0x70, 0x65, 0x63, 0x69, 0x61, 0x6c, 0x43, 0x68, 0x61, 0x72, 0x42, 0x44, 0x5a, + 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, + 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, + 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, + 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/pb/detector_typepb/detector_type.pb.go b/pkg/pb/detector_typepb/detector_type.pb.go index 51f668528ce4..21ea199c1baf 100644 --- a/pkg/pb/detector_typepb/detector_type.pb.go +++ b/pkg/pb/detector_typepb/detector_type.pb.go @@ -1001,8 +1001,9 @@ const ( DetectorType_VagrantCloudPersonalToken DetectorType = 946 DetectorType_BetterStack DetectorType = 947 DetectorType_ZeroTier DetectorType = 948 - DetectorType_AppOptics DetectorType = 949 - DetectorType_Metabase DetectorType = 950 + // Deprecated: Marked as deprecated in detector_type.proto. + DetectorType_AppOptics DetectorType = 949 + DetectorType_Metabase DetectorType = 950 // Deprecated: Marked as deprecated in detector_type.proto. DetectorType_CoinbaseWaaS DetectorType = 951 DetectorType_LemonSqueezy DetectorType = 952 @@ -1075,32 +1076,57 @@ const ( DetectorType_AzureAPIManagementSubscriptionKey DetectorType = 1019 DetectorType_Harness DetectorType = 1020 DetectorType_Langfuse DetectorType = 1021 - DetectorType_BingSubscriptionKey DetectorType = 1022 - DetectorType_XAI DetectorType = 1023 - DetectorType_AzureDirectManagementKey DetectorType = 1024 - DetectorType_AzureAppConfigConnectionString DetectorType = 1025 - DetectorType_DeepSeek DetectorType = 1026 - DetectorType_StripePaymentIntent DetectorType = 1027 - DetectorType_LangSmith DetectorType = 1028 - DetectorType_BitbucketAppPassword DetectorType = 1029 - DetectorType_Hasura DetectorType = 1030 - DetectorType_SalesforceRefreshToken DetectorType = 1031 - DetectorType_AnypointOAuth2 DetectorType = 1032 - DetectorType_WebexBot DetectorType = 1033 - DetectorType_TableauPersonalAccessToken DetectorType = 1034 - DetectorType_Rootly DetectorType = 1035 - DetectorType_HashiCorpVaultAuth DetectorType = 1036 - DetectorType_PhraseAccessToken DetectorType = 1037 - DetectorType_Photoroom DetectorType = 1038 - DetectorType_JWT DetectorType = 1039 - DetectorType_OpenAIAdmin DetectorType = 1040 - DetectorType_GoogleGeminiAPIKey DetectorType = 1041 - DetectorType_ArtifactoryReferenceToken DetectorType = 1042 - DetectorType_DatadogApikey DetectorType = 1043 - DetectorType_ShopifyOAuth DetectorType = 1044 - DetectorType_BitbucketDataCenter DetectorType = 1045 - DetectorType_JiraDataCenterPAT DetectorType = 1046 - DetectorType_ConfluenceDataCenter DetectorType = 1047 + // Deprecated: Marked as deprecated in detector_type.proto. + DetectorType_BingSubscriptionKey DetectorType = 1022 + DetectorType_XAI DetectorType = 1023 + DetectorType_AzureDirectManagementKey DetectorType = 1024 + DetectorType_AzureAppConfigConnectionString DetectorType = 1025 + DetectorType_DeepSeek DetectorType = 1026 + DetectorType_StripePaymentIntent DetectorType = 1027 + DetectorType_LangSmith DetectorType = 1028 + DetectorType_BitbucketAppPassword DetectorType = 1029 + DetectorType_Hasura DetectorType = 1030 + DetectorType_SalesforceRefreshToken DetectorType = 1031 + DetectorType_AnypointOAuth2 DetectorType = 1032 + DetectorType_WebexBot DetectorType = 1033 + DetectorType_TableauPersonalAccessToken DetectorType = 1034 + DetectorType_Rootly DetectorType = 1035 + DetectorType_HashiCorpVaultAuth DetectorType = 1036 + DetectorType_PhraseAccessToken DetectorType = 1037 + DetectorType_Photoroom DetectorType = 1038 + DetectorType_JWT DetectorType = 1039 + DetectorType_OpenAIAdmin DetectorType = 1040 + DetectorType_GoogleGeminiAPIKey DetectorType = 1041 + DetectorType_ArtifactoryReferenceToken DetectorType = 1042 + DetectorType_DatadogApikey DetectorType = 1043 + DetectorType_ShopifyOAuth DetectorType = 1044 + DetectorType_BitbucketDataCenter DetectorType = 1045 + DetectorType_JiraDataCenterPAT DetectorType = 1046 + DetectorType_ConfluenceDataCenter DetectorType = 1047 + DetectorType_Cloudinary DetectorType = 1048 + DetectorType_Pinecone DetectorType = 1049 + DetectorType_GitLabOauth2 DetectorType = 1050 + DetectorType_SpectralOps DetectorType = 1051 + DetectorType_AWSAppSync DetectorType = 1052 + DetectorType_BrainTrustApiKey DetectorType = 1053 + DetectorType_PgAnalyzeReadKey DetectorType = 1054 + DetectorType_RedHatPyxis DetectorType = 1055 + DetectorType_OctopusDeploy DetectorType = 1056 + DetectorType_OpenRouter DetectorType = 1057 + DetectorType_NewRelicInsightsInsertKey DetectorType = 1058 + DetectorType_DuffelToken DetectorType = 1059 + DetectorType_Shippo DetectorType = 1060 + DetectorType_HashiCorpVaultBatchToken DetectorType = 1061 + DetectorType_HashiCorpVaultToken DetectorType = 1062 + DetectorType_Duo DetectorType = 1063 + DetectorType_NewRelicLicenseKey DetectorType = 1064 + DetectorType_NewRelicBrowserKey DetectorType = 1065 + DetectorType_NewRelicUserKey DetectorType = 1066 + DetectorType_NewRelicInsightsQueryKey DetectorType = 1067 + DetectorType_NewRelicMobileAppToken DetectorType = 1068 + DetectorType_SolarWindsObservability DetectorType = 1069 + DetectorType_HumioAPIToken DetectorType = 1070 + DetectorType_Resend DetectorType = 1071 ) // Enum value maps for DetectorType. @@ -2150,6 +2176,30 @@ var ( 1045: "BitbucketDataCenter", 1046: "JiraDataCenterPAT", 1047: "ConfluenceDataCenter", + 1048: "Cloudinary", + 1049: "Pinecone", + 1050: "GitLabOauth2", + 1051: "SpectralOps", + 1052: "AWSAppSync", + 1053: "BrainTrustApiKey", + 1054: "PgAnalyzeReadKey", + 1055: "RedHatPyxis", + 1056: "OctopusDeploy", + 1057: "OpenRouter", + 1058: "NewRelicInsightsInsertKey", + 1059: "DuffelToken", + 1060: "Shippo", + 1061: "HashiCorpVaultBatchToken", + 1062: "HashiCorpVaultToken", + 1063: "Duo", + 1064: "NewRelicLicenseKey", + 1065: "NewRelicBrowserKey", + 1066: "NewRelicUserKey", + 1067: "NewRelicInsightsQueryKey", + 1068: "NewRelicMobileAppToken", + 1069: "SolarWindsObservability", + 1070: "HumioAPIToken", + 1071: "Resend", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -3196,6 +3246,30 @@ var ( "BitbucketDataCenter": 1045, "JiraDataCenterPAT": 1046, "ConfluenceDataCenter": 1047, + "Cloudinary": 1048, + "Pinecone": 1049, + "GitLabOauth2": 1050, + "SpectralOps": 1051, + "AWSAppSync": 1052, + "BrainTrustApiKey": 1053, + "PgAnalyzeReadKey": 1054, + "RedHatPyxis": 1055, + "OctopusDeploy": 1056, + "OpenRouter": 1057, + "NewRelicInsightsInsertKey": 1058, + "DuffelToken": 1059, + "Shippo": 1060, + "HashiCorpVaultBatchToken": 1061, + "HashiCorpVaultToken": 1062, + "Duo": 1063, + "NewRelicLicenseKey": 1064, + "NewRelicBrowserKey": 1065, + "NewRelicUserKey": 1066, + "NewRelicInsightsQueryKey": 1067, + "NewRelicMobileAppToken": 1068, + "SolarWindsObservability": 1069, + "HumioAPIToken": 1070, + "Resend": 1071, } ) @@ -3231,7 +3305,7 @@ var File_detector_type_proto protoreflect.FileDescriptor var file_detector_type_proto_rawDesc = []byte{ 0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x2a, 0x8c, 0x88, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, + 0x74, 0x79, 0x70, 0x65, 0x2a, 0x94, 0x8c, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, @@ -4191,140 +4265,173 @@ var file_detector_type_proto_rawDesc = []byte{ 0x56, 0x61, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x50, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, 0x6c, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xb2, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x42, 0x65, 0x74, 0x74, 0x65, 0x72, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x10, 0xb3, 0x07, 0x12, 0x0d, - 0x0a, 0x08, 0x5a, 0x65, 0x72, 0x6f, 0x54, 0x69, 0x65, 0x72, 0x10, 0xb4, 0x07, 0x12, 0x0e, 0x0a, - 0x09, 0x41, 0x70, 0x70, 0x4f, 0x70, 0x74, 0x69, 0x63, 0x73, 0x10, 0xb5, 0x07, 0x12, 0x0d, 0x0a, - 0x08, 0x4d, 0x65, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x10, 0xb6, 0x07, 0x12, 0x15, 0x0a, 0x0c, - 0x43, 0x6f, 0x69, 0x6e, 0x62, 0x61, 0x73, 0x65, 0x57, 0x61, 0x61, 0x53, 0x10, 0xb7, 0x07, 0x1a, - 0x02, 0x08, 0x01, 0x12, 0x11, 0x0a, 0x0c, 0x4c, 0x65, 0x6d, 0x6f, 0x6e, 0x53, 0x71, 0x75, 0x65, - 0x65, 0x7a, 0x79, 0x10, 0xb8, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x42, 0x75, 0x64, 0x69, 0x62, 0x61, - 0x73, 0x65, 0x10, 0xb9, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x44, 0x65, 0x6e, 0x6f, 0x44, 0x65, 0x70, - 0x6c, 0x6f, 0x79, 0x10, 0xba, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x53, 0x74, 0x72, 0x69, 0x70, 0x6f, - 0x10, 0xbb, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x49, 0x4f, 0x10, 0xbc, - 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x10, - 0xbd, 0x07, 0x12, 0x1b, 0x0a, 0x16, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x10, 0xbe, 0x07, 0x12, - 0x12, 0x0a, 0x0d, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, - 0x10, 0xbf, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x61, 0x10, 0xc0, 0x07, 0x12, 0x0b, - 0x0a, 0x06, 0x4c, 0x6f, 0x67, 0x7a, 0x49, 0x4f, 0x10, 0xc1, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x62, 0x72, 0x69, 0x74, 0x65, 0x10, 0xc2, 0x07, 0x12, 0x1a, 0x0a, 0x15, - 0x47, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x10, 0xc3, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x46, 0x69, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x10, 0xc4, 0x07, 0x12, 0x0d, 0x0a, - 0x08, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x6f, 0x6f, 0x70, 0x10, 0xc5, 0x07, 0x12, 0x0a, 0x0a, 0x05, - 0x4e, 0x67, 0x72, 0x6f, 0x6b, 0x10, 0xc6, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0xc7, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x50, 0x6f, 0x73, 0x74, - 0x67, 0x72, 0x65, 0x73, 0x10, 0xc8, 0x07, 0x12, 0x2a, 0x0a, 0x25, 0x41, 0x7a, 0x75, 0x72, 0x65, - 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x41, - 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x10, 0xc9, 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x46, 0x6f, 0x72, 0x52, 0x65, 0x64, 0x69, 0x73, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, - 0x65, 0x79, 0x10, 0xca, 0x07, 0x12, 0x21, 0x0a, 0x1c, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x43, 0x6f, - 0x73, 0x6d, 0x6f, 0x73, 0x44, 0x42, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, - 0x69, 0x61, 0x62, 0x6c, 0x65, 0x10, 0xcb, 0x07, 0x12, 0x23, 0x0a, 0x1e, 0x41, 0x7a, 0x75, 0x72, - 0x65, 0x44, 0x65, 0x76, 0x6f, 0x70, 0x73, 0x50, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, 0x6c, 0x41, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xcc, 0x07, 0x12, 0x15, 0x0a, - 0x10, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, - 0x79, 0x10, 0xcd, 0x07, 0x12, 0x2c, 0x0a, 0x27, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x4d, 0x4c, 0x57, - 0x65, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x63, - 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x10, - 0xce, 0x07, 0x12, 0x12, 0x0a, 0x0d, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x53, 0x61, 0x73, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x10, 0xcf, 0x07, 0x12, 0x18, 0x0a, 0x13, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x4b, 0x65, 0x79, 0x10, 0xd0, 0x07, - 0x12, 0x18, 0x0a, 0x13, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xd1, 0x07, 0x12, 0x1f, 0x0a, 0x1a, 0x41, 0x7a, - 0x75, 0x72, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, - 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0xd2, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x41, - 0x7a, 0x75, 0x72, 0x65, 0x53, 0x51, 0x4c, 0x10, 0xd3, 0x07, 0x12, 0x0a, 0x0a, 0x05, 0x46, 0x6c, - 0x79, 0x49, 0x4f, 0x10, 0xd4, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x74, 0x57, - 0x69, 0x74, 0x68, 0x10, 0xd5, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x4a, 0x75, 0x70, 0x69, 0x74, 0x65, - 0x72, 0x4f, 0x6e, 0x65, 0x10, 0xd6, 0x07, 0x12, 0x25, 0x0a, 0x20, 0x47, 0x43, 0x50, 0x41, 0x70, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, - 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x10, 0xd7, 0x07, 0x12, 0x08, - 0x0a, 0x03, 0x57, 0x69, 0x7a, 0x10, 0xd8, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x61, 0x67, 0x61, - 0x72, 0x6d, 0x65, 0x10, 0xd9, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x4f, 0x6e, 0x66, 0x6c, 0x65, 0x65, - 0x74, 0x10, 0xda, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x49, 0x6e, 0x74, 0x72, 0x61, 0x34, 0x32, 0x10, - 0xdb, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x47, 0x72, 0x6f, 0x71, 0x10, 0xdc, 0x07, 0x12, 0x17, 0x0a, - 0x12, 0x54, 0x77, 0x69, 0x74, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, - 0x6b, 0x65, 0x79, 0x10, 0xdd, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x45, 0x72, 0x61, 0x73, 0x65, 0x72, - 0x10, 0xde, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x4c, 0x61, 0x72, 0x6b, 0x53, 0x75, 0x69, 0x74, 0x65, - 0x10, 0xdf, 0x07, 0x12, 0x14, 0x0a, 0x0f, 0x4c, 0x61, 0x72, 0x6b, 0x53, 0x75, 0x69, 0x74, 0x65, - 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xe0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x45, 0x6e, 0x64, - 0x6f, 0x72, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe1, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x45, 0x6c, 0x65, - 0x76, 0x65, 0x6e, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe2, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4e, 0x65, - 0x74, 0x73, 0x75, 0x69, 0x74, 0x65, 0x10, 0xe3, 0x07, 0x12, 0x14, 0x0a, 0x0f, 0x52, 0x6f, 0x62, - 0x69, 0x6e, 0x68, 0x6f, 0x6f, 0x64, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x10, 0xe4, 0x07, 0x12, - 0x0a, 0x0a, 0x05, 0x4e, 0x56, 0x41, 0x50, 0x49, 0x10, 0xe5, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x50, - 0x79, 0x50, 0x49, 0x10, 0xe6, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x52, 0x61, 0x69, 0x6c, 0x77, 0x61, - 0x79, 0x41, 0x70, 0x70, 0x10, 0xe7, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x4d, 0x65, 0x72, 0x61, 0x6b, - 0x69, 0x10, 0xe8, 0x07, 0x12, 0x15, 0x0a, 0x10, 0x53, 0x61, 0x6c, 0x61, 0x64, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xe9, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x42, - 0x6f, 0x78, 0x10, 0xea, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x42, 0x6f, 0x78, 0x4f, 0x61, 0x75, 0x74, - 0x68, 0x10, 0xeb, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x41, 0x70, 0x69, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x10, 0xec, 0x07, 0x12, 0x15, 0x0a, 0x10, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, - 0x41, 0x6e, 0x64, 0x42, 0x69, 0x61, 0x73, 0x65, 0x73, 0x10, 0xed, 0x07, 0x12, 0x0c, 0x0a, 0x07, - 0x5a, 0x6f, 0x68, 0x6f, 0x43, 0x52, 0x4d, 0x10, 0xee, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x41, 0x7a, - 0x75, 0x72, 0x65, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x49, 0x10, 0xef, 0x07, 0x12, 0x0c, 0x0a, 0x07, - 0x47, 0x6f, 0x44, 0x61, 0x64, 0x64, 0x79, 0x10, 0xf0, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x46, 0x6c, - 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x10, 0xf1, 0x07, 0x12, 0x16, 0x0a, 0x11, 0x54, 0x77, 0x69, - 0x74, 0x63, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf2, - 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x54, 0x77, 0x69, 0x6c, 0x69, 0x6f, 0x41, 0x70, 0x69, 0x4b, 0x65, - 0x79, 0x10, 0xf3, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x53, 0x61, 0x6e, 0x69, 0x74, 0x79, 0x10, 0xf4, - 0x07, 0x12, 0x16, 0x0a, 0x11, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, - 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf5, 0x07, 0x12, 0x12, 0x0a, 0x0d, 0x41, 0x69, 0x72, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x41, 0x75, 0x74, 0x68, 0x10, 0xf6, 0x07, 0x12, 0x20, 0x0a, - 0x1b, 0x41, 0x69, 0x72, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, - 0x6c, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf7, 0x07, 0x12, - 0x21, 0x0a, 0x1c, 0x53, 0x74, 0x6f, 0x72, 0x79, 0x62, 0x6c, 0x6f, 0x6b, 0x50, 0x65, 0x72, 0x73, - 0x6f, 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, - 0xf8, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x53, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x4f, 0x72, 0x67, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf9, 0x07, 0x12, 0x24, 0x0a, 0x1f, 0x41, 0x7a, 0x75, 0x72, 0x65, - 0x41, 0x70, 0x69, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xfa, 0x07, 0x12, 0x26, 0x0a, - 0x21, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x50, 0x49, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, - 0x65, 0x79, 0x10, 0xfb, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x48, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, - 0x10, 0xfc, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4c, 0x61, 0x6e, 0x67, 0x66, 0x75, 0x73, 0x65, 0x10, - 0xfd, 0x07, 0x12, 0x18, 0x0a, 0x13, 0x42, 0x69, 0x6e, 0x67, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x10, 0xfe, 0x07, 0x12, 0x08, 0x0a, 0x03, - 0x58, 0x41, 0x49, 0x10, 0xff, 0x07, 0x12, 0x1d, 0x0a, 0x18, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x44, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4b, - 0x65, 0x79, 0x10, 0x80, 0x08, 0x12, 0x23, 0x0a, 0x1e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x70, - 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x10, 0x81, 0x08, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x65, - 0x65, 0x70, 0x53, 0x65, 0x65, 0x6b, 0x10, 0x82, 0x08, 0x12, 0x18, 0x0a, 0x13, 0x53, 0x74, 0x72, - 0x69, 0x70, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x10, 0x83, 0x08, 0x12, 0x0e, 0x0a, 0x09, 0x4c, 0x61, 0x6e, 0x67, 0x53, 0x6d, 0x69, 0x74, 0x68, - 0x10, 0x84, 0x08, 0x12, 0x19, 0x0a, 0x14, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x41, 0x70, 0x70, 0x50, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x10, 0x85, 0x08, 0x12, 0x0b, - 0x0a, 0x06, 0x48, 0x61, 0x73, 0x75, 0x72, 0x61, 0x10, 0x86, 0x08, 0x12, 0x1b, 0x0a, 0x16, 0x53, - 0x61, 0x6c, 0x65, 0x73, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x87, 0x08, 0x12, 0x13, 0x0a, 0x0e, 0x41, 0x6e, 0x79, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x4f, 0x41, 0x75, 0x74, 0x68, 0x32, 0x10, 0x88, 0x08, 0x12, 0x0d, 0x0a, - 0x08, 0x57, 0x65, 0x62, 0x65, 0x78, 0x42, 0x6f, 0x74, 0x10, 0x89, 0x08, 0x12, 0x1f, 0x0a, 0x1a, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x61, 0x75, 0x50, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, 0x6c, 0x41, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x8a, 0x08, 0x12, 0x0b, 0x0a, - 0x06, 0x52, 0x6f, 0x6f, 0x74, 0x6c, 0x79, 0x10, 0x8b, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x48, 0x61, - 0x73, 0x68, 0x69, 0x43, 0x6f, 0x72, 0x70, 0x56, 0x61, 0x75, 0x6c, 0x74, 0x41, 0x75, 0x74, 0x68, - 0x10, 0x8c, 0x08, 0x12, 0x16, 0x0a, 0x11, 0x50, 0x68, 0x72, 0x61, 0x73, 0x65, 0x41, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x8d, 0x08, 0x12, 0x0e, 0x0a, 0x09, 0x50, - 0x68, 0x6f, 0x74, 0x6f, 0x72, 0x6f, 0x6f, 0x6d, 0x10, 0x8e, 0x08, 0x12, 0x08, 0x0a, 0x03, 0x4a, - 0x57, 0x54, 0x10, 0x8f, 0x08, 0x12, 0x10, 0x0a, 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x49, 0x41, - 0x64, 0x6d, 0x69, 0x6e, 0x10, 0x90, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x47, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x47, 0x65, 0x6d, 0x69, 0x6e, 0x69, 0x41, 0x50, 0x49, 0x4b, 0x65, 0x79, 0x10, 0x91, 0x08, - 0x12, 0x1e, 0x0a, 0x19, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x92, 0x08, - 0x12, 0x12, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x41, 0x70, 0x69, 0x6b, 0x65, - 0x79, 0x10, 0x93, 0x08, 0x12, 0x11, 0x0a, 0x0c, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66, 0x79, 0x4f, - 0x41, 0x75, 0x74, 0x68, 0x10, 0x94, 0x08, 0x12, 0x18, 0x0a, 0x13, 0x42, 0x69, 0x74, 0x62, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x10, 0x95, - 0x08, 0x12, 0x16, 0x0a, 0x11, 0x4a, 0x69, 0x72, 0x61, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, - 0x74, 0x65, 0x72, 0x50, 0x41, 0x54, 0x10, 0x96, 0x08, 0x12, 0x19, 0x0a, 0x14, 0x43, 0x6f, 0x6e, - 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, - 0x72, 0x10, 0x97, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0a, 0x08, 0x5a, 0x65, 0x72, 0x6f, 0x54, 0x69, 0x65, 0x72, 0x10, 0xb4, 0x07, 0x12, 0x12, 0x0a, + 0x09, 0x41, 0x70, 0x70, 0x4f, 0x70, 0x74, 0x69, 0x63, 0x73, 0x10, 0xb5, 0x07, 0x1a, 0x02, 0x08, + 0x01, 0x12, 0x0d, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x10, 0xb6, 0x07, + 0x12, 0x15, 0x0a, 0x0c, 0x43, 0x6f, 0x69, 0x6e, 0x62, 0x61, 0x73, 0x65, 0x57, 0x61, 0x61, 0x53, + 0x10, 0xb7, 0x07, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x11, 0x0a, 0x0c, 0x4c, 0x65, 0x6d, 0x6f, 0x6e, + 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x79, 0x10, 0xb8, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x42, 0x75, + 0x64, 0x69, 0x62, 0x61, 0x73, 0x65, 0x10, 0xb9, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x44, 0x65, 0x6e, + 0x6f, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x10, 0xba, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x53, 0x74, + 0x72, 0x69, 0x70, 0x6f, 0x10, 0xbb, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x49, 0x4f, 0x10, 0xbc, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x42, 0x61, + 0x74, 0x63, 0x68, 0x10, 0xbd, 0x07, 0x12, 0x1b, 0x0a, 0x16, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, + 0x10, 0xbe, 0x07, 0x12, 0x12, 0x0a, 0x0d, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x4b, 0x65, 0x79, 0x10, 0xbf, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x61, 0x10, + 0xc0, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x4c, 0x6f, 0x67, 0x7a, 0x49, 0x4f, 0x10, 0xc1, 0x07, 0x12, + 0x0f, 0x0a, 0x0a, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x62, 0x72, 0x69, 0x74, 0x65, 0x10, 0xc2, 0x07, + 0x12, 0x1a, 0x0a, 0x15, 0x47, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x10, 0xc3, 0x07, 0x12, 0x13, 0x0a, 0x0e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x46, 0x69, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x10, 0xc4, + 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x6f, 0x6f, 0x70, 0x10, 0xc5, 0x07, + 0x12, 0x0a, 0x0a, 0x05, 0x4e, 0x67, 0x72, 0x6f, 0x6b, 0x10, 0xc6, 0x07, 0x12, 0x0e, 0x0a, 0x09, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0xc7, 0x07, 0x12, 0x0d, 0x0a, 0x08, + 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x10, 0xc8, 0x07, 0x12, 0x2a, 0x0a, 0x25, 0x41, + 0x7a, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x10, 0xc9, 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x41, 0x7a, 0x75, 0x72, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x46, 0x6f, 0x72, 0x52, 0x65, 0x64, 0x69, 0x73, 0x41, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x10, 0xca, 0x07, 0x12, 0x21, 0x0a, 0x1c, 0x41, 0x7a, 0x75, + 0x72, 0x65, 0x43, 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x44, 0x42, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x66, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x10, 0xcb, 0x07, 0x12, 0x23, 0x0a, 0x1e, + 0x41, 0x7a, 0x75, 0x72, 0x65, 0x44, 0x65, 0x76, 0x6f, 0x70, 0x73, 0x50, 0x65, 0x72, 0x73, 0x6f, + 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xcc, + 0x07, 0x12, 0x15, 0x0a, 0x10, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x10, 0xcd, 0x07, 0x12, 0x2c, 0x0a, 0x27, 0x41, 0x7a, 0x75, 0x72, + 0x65, 0x4d, 0x4c, 0x57, 0x65, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6c, 0x61, + 0x73, 0x73, 0x69, 0x63, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x61, 0x62, 0x6c, 0x65, + 0x4b, 0x65, 0x79, 0x10, 0xce, 0x07, 0x12, 0x12, 0x0a, 0x0d, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x53, + 0x61, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xcf, 0x07, 0x12, 0x18, 0x0a, 0x13, 0x41, 0x7a, + 0x75, 0x72, 0x65, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x4b, 0x65, + 0x79, 0x10, 0xd0, 0x07, 0x12, 0x18, 0x0a, 0x13, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xd1, 0x07, 0x12, 0x1f, + 0x0a, 0x1a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0xd2, 0x07, 0x12, + 0x0d, 0x0a, 0x08, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x53, 0x51, 0x4c, 0x10, 0xd3, 0x07, 0x12, 0x0a, + 0x0a, 0x05, 0x46, 0x6c, 0x79, 0x49, 0x4f, 0x10, 0xd4, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x42, 0x75, + 0x69, 0x6c, 0x74, 0x57, 0x69, 0x74, 0x68, 0x10, 0xd5, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x4a, 0x75, + 0x70, 0x69, 0x74, 0x65, 0x72, 0x4f, 0x6e, 0x65, 0x10, 0xd6, 0x07, 0x12, 0x25, 0x0a, 0x20, 0x47, + 0x43, 0x50, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x66, + 0x61, 0x75, 0x6c, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x10, + 0xd7, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x57, 0x69, 0x7a, 0x10, 0xd8, 0x07, 0x12, 0x0c, 0x0a, 0x07, + 0x50, 0x61, 0x67, 0x61, 0x72, 0x6d, 0x65, 0x10, 0xd9, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x4f, 0x6e, + 0x66, 0x6c, 0x65, 0x65, 0x74, 0x10, 0xda, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x49, 0x6e, 0x74, 0x72, + 0x61, 0x34, 0x32, 0x10, 0xdb, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x47, 0x72, 0x6f, 0x71, 0x10, 0xdc, + 0x07, 0x12, 0x17, 0x0a, 0x12, 0x54, 0x77, 0x69, 0x74, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x73, + 0x75, 0x6d, 0x65, 0x72, 0x6b, 0x65, 0x79, 0x10, 0xdd, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x45, 0x72, + 0x61, 0x73, 0x65, 0x72, 0x10, 0xde, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x4c, 0x61, 0x72, 0x6b, 0x53, + 0x75, 0x69, 0x74, 0x65, 0x10, 0xdf, 0x07, 0x12, 0x14, 0x0a, 0x0f, 0x4c, 0x61, 0x72, 0x6b, 0x53, + 0x75, 0x69, 0x74, 0x65, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xe0, 0x07, 0x12, 0x0e, 0x0a, + 0x09, 0x45, 0x6e, 0x64, 0x6f, 0x72, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe1, 0x07, 0x12, 0x0f, 0x0a, + 0x0a, 0x45, 0x6c, 0x65, 0x76, 0x65, 0x6e, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe2, 0x07, 0x12, 0x0d, + 0x0a, 0x08, 0x4e, 0x65, 0x74, 0x73, 0x75, 0x69, 0x74, 0x65, 0x10, 0xe3, 0x07, 0x12, 0x14, 0x0a, + 0x0f, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x68, 0x6f, 0x6f, 0x64, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, + 0x10, 0xe4, 0x07, 0x12, 0x0a, 0x0a, 0x05, 0x4e, 0x56, 0x41, 0x50, 0x49, 0x10, 0xe5, 0x07, 0x12, + 0x09, 0x0a, 0x04, 0x50, 0x79, 0x50, 0x49, 0x10, 0xe6, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x52, 0x61, + 0x69, 0x6c, 0x77, 0x61, 0x79, 0x41, 0x70, 0x70, 0x10, 0xe7, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x4d, + 0x65, 0x72, 0x61, 0x6b, 0x69, 0x10, 0xe8, 0x07, 0x12, 0x15, 0x0a, 0x10, 0x53, 0x61, 0x6c, 0x61, + 0x64, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xe9, 0x07, 0x12, + 0x08, 0x0a, 0x03, 0x42, 0x6f, 0x78, 0x10, 0xea, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x42, 0x6f, 0x78, + 0x4f, 0x61, 0x75, 0x74, 0x68, 0x10, 0xeb, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x41, 0x70, 0x69, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x10, 0xec, 0x07, 0x12, 0x15, 0x0a, 0x10, 0x57, 0x65, 0x69, + 0x67, 0x68, 0x74, 0x73, 0x41, 0x6e, 0x64, 0x42, 0x69, 0x61, 0x73, 0x65, 0x73, 0x10, 0xed, 0x07, + 0x12, 0x0c, 0x0a, 0x07, 0x5a, 0x6f, 0x68, 0x6f, 0x43, 0x52, 0x4d, 0x10, 0xee, 0x07, 0x12, 0x10, + 0x0a, 0x0b, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x49, 0x10, 0xef, 0x07, + 0x12, 0x0c, 0x0a, 0x07, 0x47, 0x6f, 0x44, 0x61, 0x64, 0x64, 0x79, 0x10, 0xf0, 0x07, 0x12, 0x0d, + 0x0a, 0x08, 0x46, 0x6c, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x10, 0xf1, 0x07, 0x12, 0x16, 0x0a, + 0x11, 0x54, 0x77, 0x69, 0x74, 0x63, 0x68, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x10, 0xf2, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x54, 0x77, 0x69, 0x6c, 0x69, 0x6f, 0x41, + 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xf3, 0x07, 0x12, 0x0b, 0x0a, 0x06, 0x53, 0x61, 0x6e, 0x69, + 0x74, 0x79, 0x10, 0xf4, 0x07, 0x12, 0x16, 0x0a, 0x11, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf5, 0x07, 0x12, 0x12, 0x0a, + 0x0d, 0x41, 0x69, 0x72, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x41, 0x75, 0x74, 0x68, 0x10, 0xf6, + 0x07, 0x12, 0x20, 0x0a, 0x1b, 0x41, 0x69, 0x72, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x65, 0x72, + 0x73, 0x6f, 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x10, 0xf7, 0x07, 0x12, 0x21, 0x0a, 0x1c, 0x53, 0x74, 0x6f, 0x72, 0x79, 0x62, 0x6c, 0x6f, 0x6b, + 0x50, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x10, 0xf8, 0x07, 0x12, 0x13, 0x0a, 0x0e, 0x53, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x4f, 0x72, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xf9, 0x07, 0x12, 0x24, 0x0a, 0x1f, 0x41, + 0x7a, 0x75, 0x72, 0x65, 0x41, 0x70, 0x69, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xfa, + 0x07, 0x12, 0x26, 0x0a, 0x21, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x50, 0x49, 0x4d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x10, 0xfb, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x48, 0x61, 0x72, + 0x6e, 0x65, 0x73, 0x73, 0x10, 0xfc, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4c, 0x61, 0x6e, 0x67, 0x66, + 0x75, 0x73, 0x65, 0x10, 0xfd, 0x07, 0x12, 0x1c, 0x0a, 0x13, 0x42, 0x69, 0x6e, 0x67, 0x53, 0x75, + 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x10, 0xfe, 0x07, + 0x1a, 0x02, 0x08, 0x01, 0x12, 0x08, 0x0a, 0x03, 0x58, 0x41, 0x49, 0x10, 0xff, 0x07, 0x12, 0x1d, + 0x0a, 0x18, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x10, 0x80, 0x08, 0x12, 0x23, 0x0a, + 0x1e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x41, 0x70, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x10, + 0x81, 0x08, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x65, 0x65, 0x70, 0x53, 0x65, 0x65, 0x6b, 0x10, 0x82, + 0x08, 0x12, 0x18, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x69, 0x70, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x10, 0x83, 0x08, 0x12, 0x0e, 0x0a, 0x09, 0x4c, + 0x61, 0x6e, 0x67, 0x53, 0x6d, 0x69, 0x74, 0x68, 0x10, 0x84, 0x08, 0x12, 0x19, 0x0a, 0x14, 0x42, + 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x41, 0x70, 0x70, 0x50, 0x61, 0x73, 0x73, 0x77, + 0x6f, 0x72, 0x64, 0x10, 0x85, 0x08, 0x12, 0x0b, 0x0a, 0x06, 0x48, 0x61, 0x73, 0x75, 0x72, 0x61, + 0x10, 0x86, 0x08, 0x12, 0x1b, 0x0a, 0x16, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66, 0x6f, 0x72, 0x63, + 0x65, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x87, 0x08, + 0x12, 0x13, 0x0a, 0x0e, 0x41, 0x6e, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4f, 0x41, 0x75, 0x74, + 0x68, 0x32, 0x10, 0x88, 0x08, 0x12, 0x0d, 0x0a, 0x08, 0x57, 0x65, 0x62, 0x65, 0x78, 0x42, 0x6f, + 0x74, 0x10, 0x89, 0x08, 0x12, 0x1f, 0x0a, 0x1a, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x61, 0x75, 0x50, + 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x10, 0x8a, 0x08, 0x12, 0x0b, 0x0a, 0x06, 0x52, 0x6f, 0x6f, 0x74, 0x6c, 0x79, 0x10, + 0x8b, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x48, 0x61, 0x73, 0x68, 0x69, 0x43, 0x6f, 0x72, 0x70, 0x56, + 0x61, 0x75, 0x6c, 0x74, 0x41, 0x75, 0x74, 0x68, 0x10, 0x8c, 0x08, 0x12, 0x16, 0x0a, 0x11, 0x50, + 0x68, 0x72, 0x61, 0x73, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x10, 0x8d, 0x08, 0x12, 0x0e, 0x0a, 0x09, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x72, 0x6f, 0x6f, 0x6d, + 0x10, 0x8e, 0x08, 0x12, 0x08, 0x0a, 0x03, 0x4a, 0x57, 0x54, 0x10, 0x8f, 0x08, 0x12, 0x10, 0x0a, + 0x0b, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x49, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x10, 0x90, 0x08, 0x12, + 0x17, 0x0a, 0x12, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x47, 0x65, 0x6d, 0x69, 0x6e, 0x69, 0x41, + 0x50, 0x49, 0x4b, 0x65, 0x79, 0x10, 0x91, 0x08, 0x12, 0x1e, 0x0a, 0x19, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x92, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, + 0x64, 0x6f, 0x67, 0x41, 0x70, 0x69, 0x6b, 0x65, 0x79, 0x10, 0x93, 0x08, 0x12, 0x11, 0x0a, 0x0c, + 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66, 0x79, 0x4f, 0x41, 0x75, 0x74, 0x68, 0x10, 0x94, 0x08, 0x12, + 0x18, 0x0a, 0x13, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, + 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x10, 0x95, 0x08, 0x12, 0x16, 0x0a, 0x11, 0x4a, 0x69, 0x72, + 0x61, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x50, 0x41, 0x54, 0x10, 0x96, + 0x08, 0x12, 0x19, 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x44, + 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x10, 0x97, 0x08, 0x12, 0x0f, 0x0a, 0x0a, + 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x10, 0x98, 0x08, 0x12, 0x0d, 0x0a, + 0x08, 0x50, 0x69, 0x6e, 0x65, 0x63, 0x6f, 0x6e, 0x65, 0x10, 0x99, 0x08, 0x12, 0x11, 0x0a, 0x0c, + 0x47, 0x69, 0x74, 0x4c, 0x61, 0x62, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x10, 0x9a, 0x08, 0x12, + 0x10, 0x0a, 0x0b, 0x53, 0x70, 0x65, 0x63, 0x74, 0x72, 0x61, 0x6c, 0x4f, 0x70, 0x73, 0x10, 0x9b, + 0x08, 0x12, 0x0f, 0x0a, 0x0a, 0x41, 0x57, 0x53, 0x41, 0x70, 0x70, 0x53, 0x79, 0x6e, 0x63, 0x10, + 0x9c, 0x08, 0x12, 0x15, 0x0a, 0x10, 0x42, 0x72, 0x61, 0x69, 0x6e, 0x54, 0x72, 0x75, 0x73, 0x74, + 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0x9d, 0x08, 0x12, 0x15, 0x0a, 0x10, 0x50, 0x67, 0x41, + 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x52, 0x65, 0x61, 0x64, 0x4b, 0x65, 0x79, 0x10, 0x9e, 0x08, + 0x12, 0x10, 0x0a, 0x0b, 0x52, 0x65, 0x64, 0x48, 0x61, 0x74, 0x50, 0x79, 0x78, 0x69, 0x73, 0x10, + 0x9f, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x4f, 0x63, 0x74, 0x6f, 0x70, 0x75, 0x73, 0x44, 0x65, 0x70, + 0x6c, 0x6f, 0x79, 0x10, 0xa0, 0x08, 0x12, 0x0f, 0x0a, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x72, 0x10, 0xa1, 0x08, 0x12, 0x1e, 0x0a, 0x19, 0x4e, 0x65, 0x77, 0x52, 0x65, + 0x6c, 0x69, 0x63, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x49, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x4b, 0x65, 0x79, 0x10, 0xa2, 0x08, 0x12, 0x10, 0x0a, 0x0b, 0x44, 0x75, 0x66, 0x66, 0x65, + 0x6c, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xa3, 0x08, 0x12, 0x0b, 0x0a, 0x06, 0x53, 0x68, 0x69, + 0x70, 0x70, 0x6f, 0x10, 0xa4, 0x08, 0x12, 0x1d, 0x0a, 0x18, 0x48, 0x61, 0x73, 0x68, 0x69, 0x43, + 0x6f, 0x72, 0x70, 0x56, 0x61, 0x75, 0x6c, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x10, 0xa5, 0x08, 0x12, 0x18, 0x0a, 0x13, 0x48, 0x61, 0x73, 0x68, 0x69, 0x43, 0x6f, + 0x72, 0x70, 0x56, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xa6, 0x08, 0x12, + 0x08, 0x0a, 0x03, 0x44, 0x75, 0x6f, 0x10, 0xa7, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x4e, 0x65, 0x77, + 0x52, 0x65, 0x6c, 0x69, 0x63, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x10, + 0xa8, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x42, 0x72, + 0x6f, 0x77, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xa9, 0x08, 0x12, 0x14, 0x0a, 0x0f, 0x4e, + 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x55, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xaa, + 0x08, 0x12, 0x1d, 0x0a, 0x18, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x49, 0x6e, 0x73, + 0x69, 0x67, 0x68, 0x74, 0x73, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xab, 0x08, + 0x12, 0x1b, 0x0a, 0x16, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x4d, 0x6f, 0x62, 0x69, + 0x6c, 0x65, 0x41, 0x70, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xac, 0x08, 0x12, 0x1c, 0x0a, + 0x17, 0x53, 0x6f, 0x6c, 0x61, 0x72, 0x57, 0x69, 0x6e, 0x64, 0x73, 0x4f, 0x62, 0x73, 0x65, 0x72, + 0x76, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x10, 0xad, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x48, + 0x75, 0x6d, 0x69, 0x6f, 0x41, 0x50, 0x49, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xae, 0x08, 0x12, + 0x0b, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x10, 0xaf, 0x08, 0x42, 0x41, 0x5a, 0x3f, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, + 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, + 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, + 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.go b/pkg/pb/source_metadatapb/source_metadata.pb.go index a457dee1e500..9d09b88bc7f0 100644 --- a/pkg/pb/source_metadatapb/source_metadata.pb.go +++ b/pkg/pb/source_metadatapb/source_metadata.pb.go @@ -7,6 +7,7 @@ package source_metadatapb import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" @@ -167,6 +168,53 @@ func (PostmanLocationType) EnumDescriptor() ([]byte, []int) { return file_source_metadata_proto_rawDescGZIP(), []int{1} } +type AWSSecretsManager struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SecretArn string `protobuf:"bytes,1,opt,name=secret_arn,json=secretArn,proto3" json:"secret_arn,omitempty"` +} + +func (x *AWSSecretsManager) Reset() { + *x = AWSSecretsManager{} + if protoimpl.UnsafeEnabled { + mi := &file_source_metadata_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AWSSecretsManager) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AWSSecretsManager) ProtoMessage() {} + +func (x *AWSSecretsManager) ProtoReflect() protoreflect.Message { + mi := &file_source_metadata_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AWSSecretsManager.ProtoReflect.Descriptor instead. +func (*AWSSecretsManager) Descriptor() ([]byte, []int) { + return file_source_metadata_proto_rawDescGZIP(), []int{0} +} + +func (x *AWSSecretsManager) GetSecretArn() string { + if x != nil { + return x.SecretArn + } + return "" +} + type Azure struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -182,7 +230,7 @@ type Azure struct { func (x *Azure) Reset() { *x = Azure{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[0] + mi := &file_source_metadata_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -195,7 +243,7 @@ func (x *Azure) String() string { func (*Azure) ProtoMessage() {} func (x *Azure) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[0] + mi := &file_source_metadata_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -208,7 +256,7 @@ func (x *Azure) ProtoReflect() protoreflect.Message { // Deprecated: Use Azure.ProtoReflect.Descriptor instead. func (*Azure) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{0} + return file_source_metadata_proto_rawDescGZIP(), []int{1} } func (x *Azure) GetContainer() string { @@ -266,7 +314,7 @@ type Bitbucket struct { func (x *Bitbucket) Reset() { *x = Bitbucket{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[1] + mi := &file_source_metadata_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -279,7 +327,7 @@ func (x *Bitbucket) String() string { func (*Bitbucket) ProtoMessage() {} func (x *Bitbucket) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[1] + mi := &file_source_metadata_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -292,7 +340,7 @@ func (x *Bitbucket) ProtoReflect() protoreflect.Message { // Deprecated: Use Bitbucket.ProtoReflect.Descriptor instead. func (*Bitbucket) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{1} + return file_source_metadata_proto_rawDescGZIP(), []int{2} } func (x *Bitbucket) GetFile() string { @@ -381,7 +429,7 @@ type Buildkite struct { func (x *Buildkite) Reset() { *x = Buildkite{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[2] + mi := &file_source_metadata_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -394,7 +442,7 @@ func (x *Buildkite) String() string { func (*Buildkite) ProtoMessage() {} func (x *Buildkite) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[2] + mi := &file_source_metadata_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -407,7 +455,7 @@ func (x *Buildkite) ProtoReflect() protoreflect.Message { // Deprecated: Use Buildkite.ProtoReflect.Descriptor instead. func (*Buildkite) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{2} + return file_source_metadata_proto_rawDescGZIP(), []int{3} } func (x *Buildkite) GetOrg() string { @@ -469,7 +517,7 @@ type CircleCI struct { func (x *CircleCI) Reset() { *x = CircleCI{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[3] + mi := &file_source_metadata_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -482,7 +530,7 @@ func (x *CircleCI) String() string { func (*CircleCI) ProtoMessage() {} func (x *CircleCI) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[3] + mi := &file_source_metadata_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -495,7 +543,7 @@ func (x *CircleCI) ProtoReflect() protoreflect.Message { // Deprecated: Use CircleCI.ProtoReflect.Descriptor instead. func (*CircleCI) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{3} + return file_source_metadata_proto_rawDescGZIP(), []int{4} } func (x *CircleCI) GetVcsType() string { @@ -563,7 +611,7 @@ type TravisCI struct { func (x *TravisCI) Reset() { *x = TravisCI{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[4] + mi := &file_source_metadata_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -576,7 +624,7 @@ func (x *TravisCI) String() string { func (*TravisCI) ProtoMessage() {} func (x *TravisCI) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[4] + mi := &file_source_metadata_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -589,7 +637,7 @@ func (x *TravisCI) ProtoReflect() protoreflect.Message { // Deprecated: Use TravisCI.ProtoReflect.Descriptor instead. func (*TravisCI) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{4} + return file_source_metadata_proto_rawDescGZIP(), []int{5} } func (x *TravisCI) GetUsername() string { @@ -654,7 +702,7 @@ type Confluence struct { func (x *Confluence) Reset() { *x = Confluence{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[5] + mi := &file_source_metadata_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -667,7 +715,7 @@ func (x *Confluence) String() string { func (*Confluence) ProtoMessage() {} func (x *Confluence) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[5] + mi := &file_source_metadata_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -680,7 +728,7 @@ func (x *Confluence) ProtoReflect() protoreflect.Message { // Deprecated: Use Confluence.ProtoReflect.Descriptor instead. func (*Confluence) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{5} + return file_source_metadata_proto_rawDescGZIP(), []int{6} } func (x *Confluence) GetPage() string { @@ -767,7 +815,7 @@ type Docker struct { func (x *Docker) Reset() { *x = Docker{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[6] + mi := &file_source_metadata_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -780,7 +828,7 @@ func (x *Docker) String() string { func (*Docker) ProtoMessage() {} func (x *Docker) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[6] + mi := &file_source_metadata_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -793,7 +841,7 @@ func (x *Docker) ProtoReflect() protoreflect.Message { // Deprecated: Use Docker.ProtoReflect.Descriptor instead. func (*Docker) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{6} + return file_source_metadata_proto_rawDescGZIP(), []int{7} } func (x *Docker) GetFile() string { @@ -841,7 +889,7 @@ type ECR struct { func (x *ECR) Reset() { *x = ECR{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[7] + mi := &file_source_metadata_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -854,7 +902,7 @@ func (x *ECR) String() string { func (*ECR) ProtoMessage() {} func (x *ECR) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[7] + mi := &file_source_metadata_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -867,7 +915,7 @@ func (x *ECR) ProtoReflect() protoreflect.Message { // Deprecated: Use ECR.ProtoReflect.Descriptor instead. func (*ECR) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{7} + return file_source_metadata_proto_rawDescGZIP(), []int{8} } func (x *ECR) GetFile() string { @@ -933,7 +981,7 @@ type Filesystem struct { func (x *Filesystem) Reset() { *x = Filesystem{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[8] + mi := &file_source_metadata_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -946,7 +994,7 @@ func (x *Filesystem) String() string { func (*Filesystem) ProtoMessage() {} func (x *Filesystem) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[8] + mi := &file_source_metadata_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -959,7 +1007,7 @@ func (x *Filesystem) ProtoReflect() protoreflect.Message { // Deprecated: Use Filesystem.ProtoReflect.Descriptor instead. func (*Filesystem) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{8} + return file_source_metadata_proto_rawDescGZIP(), []int{9} } func (x *Filesystem) GetFile() string { @@ -1007,7 +1055,7 @@ type Git struct { func (x *Git) Reset() { *x = Git{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[9] + mi := &file_source_metadata_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1020,7 +1068,7 @@ func (x *Git) String() string { func (*Git) ProtoMessage() {} func (x *Git) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[9] + mi := &file_source_metadata_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1033,7 +1081,7 @@ func (x *Git) ProtoReflect() protoreflect.Message { // Deprecated: Use Git.ProtoReflect.Descriptor instead. func (*Git) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{9} + return file_source_metadata_proto_rawDescGZIP(), []int{10} } func (x *Git) GetCommit() string { @@ -1105,7 +1153,7 @@ type Github struct { func (x *Github) Reset() { *x = Github{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[10] + mi := &file_source_metadata_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1118,7 +1166,7 @@ func (x *Github) String() string { func (*Github) ProtoMessage() {} func (x *Github) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[10] + mi := &file_source_metadata_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1131,7 +1179,7 @@ func (x *Github) ProtoReflect() protoreflect.Message { // Deprecated: Use Github.ProtoReflect.Descriptor instead. func (*Github) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{10} + return file_source_metadata_proto_rawDescGZIP(), []int{11} } func (x *Github) GetLink() string { @@ -1225,7 +1273,7 @@ type Gitlab struct { func (x *Gitlab) Reset() { *x = Gitlab{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[11] + mi := &file_source_metadata_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1238,7 +1286,7 @@ func (x *Gitlab) String() string { func (*Gitlab) ProtoMessage() {} func (x *Gitlab) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[11] + mi := &file_source_metadata_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1251,7 +1299,7 @@ func (x *Gitlab) ProtoReflect() protoreflect.Message { // Deprecated: Use Gitlab.ProtoReflect.Descriptor instead. func (*Gitlab) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{11} + return file_source_metadata_proto_rawDescGZIP(), []int{12} } func (x *Gitlab) GetCommit() string { @@ -1349,7 +1397,7 @@ type GCS struct { func (x *GCS) Reset() { *x = GCS{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[12] + mi := &file_source_metadata_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1362,7 +1410,7 @@ func (x *GCS) String() string { func (*GCS) ProtoMessage() {} func (x *GCS) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[12] + mi := &file_source_metadata_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1375,7 +1423,7 @@ func (x *GCS) ProtoReflect() protoreflect.Message { // Deprecated: Use GCS.ProtoReflect.Descriptor instead. func (*GCS) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{12} + return file_source_metadata_proto_rawDescGZIP(), []int{13} } func (x *GCS) GetBucket() string { @@ -1454,7 +1502,7 @@ type Huggingface struct { func (x *Huggingface) Reset() { *x = Huggingface{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[13] + mi := &file_source_metadata_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1467,7 +1515,7 @@ func (x *Huggingface) String() string { func (*Huggingface) ProtoMessage() {} func (x *Huggingface) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[13] + mi := &file_source_metadata_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1480,7 +1528,7 @@ func (x *Huggingface) ProtoReflect() protoreflect.Message { // Deprecated: Use Huggingface.ProtoReflect.Descriptor instead. func (*Huggingface) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{13} + return file_source_metadata_proto_rawDescGZIP(), []int{14} } func (x *Huggingface) GetLink() string { @@ -1569,7 +1617,7 @@ type Jira struct { func (x *Jira) Reset() { *x = Jira{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[14] + mi := &file_source_metadata_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1582,7 +1630,7 @@ func (x *Jira) String() string { func (*Jira) ProtoMessage() {} func (x *Jira) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[14] + mi := &file_source_metadata_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1595,7 +1643,7 @@ func (x *Jira) ProtoReflect() protoreflect.Message { // Deprecated: Use Jira.ProtoReflect.Descriptor instead. func (*Jira) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{14} + return file_source_metadata_proto_rawDescGZIP(), []int{15} } func (x *Jira) GetIssue() string { @@ -1655,7 +1703,7 @@ type NPM struct { func (x *NPM) Reset() { *x = NPM{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[15] + mi := &file_source_metadata_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1668,7 +1716,7 @@ func (x *NPM) String() string { func (*NPM) ProtoMessage() {} func (x *NPM) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[15] + mi := &file_source_metadata_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1681,7 +1729,7 @@ func (x *NPM) ProtoReflect() protoreflect.Message { // Deprecated: Use NPM.ProtoReflect.Descriptor instead. func (*NPM) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{15} + return file_source_metadata_proto_rawDescGZIP(), []int{16} } func (x *NPM) GetFile() string { @@ -1734,7 +1782,7 @@ type PyPi struct { func (x *PyPi) Reset() { *x = PyPi{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[16] + mi := &file_source_metadata_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1747,7 +1795,7 @@ func (x *PyPi) String() string { func (*PyPi) ProtoMessage() {} func (x *PyPi) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[16] + mi := &file_source_metadata_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1760,7 +1808,7 @@ func (x *PyPi) ProtoReflect() protoreflect.Message { // Deprecated: Use PyPi.ProtoReflect.Descriptor instead. func (*PyPi) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{16} + return file_source_metadata_proto_rawDescGZIP(), []int{17} } func (x *PyPi) GetFile() string { @@ -1813,7 +1861,7 @@ type S3 struct { func (x *S3) Reset() { *x = S3{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[17] + mi := &file_source_metadata_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1826,7 +1874,7 @@ func (x *S3) String() string { func (*S3) ProtoMessage() {} func (x *S3) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[17] + mi := &file_source_metadata_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1839,7 +1887,7 @@ func (x *S3) ProtoReflect() protoreflect.Message { // Deprecated: Use S3.ProtoReflect.Descriptor instead. func (*S3) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{17} + return file_source_metadata_proto_rawDescGZIP(), []int{18} } func (x *S3) GetBucket() string { @@ -1896,7 +1944,7 @@ type Slack struct { func (x *Slack) Reset() { *x = Slack{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[18] + mi := &file_source_metadata_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1909,7 +1957,7 @@ func (x *Slack) String() string { func (*Slack) ProtoMessage() {} func (x *Slack) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[18] + mi := &file_source_metadata_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1922,7 +1970,7 @@ func (x *Slack) ProtoReflect() protoreflect.Message { // Deprecated: Use Slack.ProtoReflect.Descriptor instead. func (*Slack) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{18} + return file_source_metadata_proto_rawDescGZIP(), []int{19} } func (x *Slack) GetChannelId() string { @@ -2004,7 +2052,7 @@ type Gerrit struct { func (x *Gerrit) Reset() { *x = Gerrit{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[19] + mi := &file_source_metadata_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2017,7 +2065,7 @@ func (x *Gerrit) String() string { func (*Gerrit) ProtoMessage() {} func (x *Gerrit) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[19] + mi := &file_source_metadata_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2030,7 +2078,7 @@ func (x *Gerrit) ProtoReflect() protoreflect.Message { // Deprecated: Use Gerrit.ProtoReflect.Descriptor instead. func (*Gerrit) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{19} + return file_source_metadata_proto_rawDescGZIP(), []int{20} } func (x *Gerrit) GetCommit() string { @@ -2086,7 +2134,7 @@ type Test struct { func (x *Test) Reset() { *x = Test{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[20] + mi := &file_source_metadata_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2099,7 +2147,7 @@ func (x *Test) String() string { func (*Test) ProtoMessage() {} func (x *Test) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[20] + mi := &file_source_metadata_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2112,7 +2160,7 @@ func (x *Test) ProtoReflect() protoreflect.Message { // Deprecated: Use Test.ProtoReflect.Descriptor instead. func (*Test) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{20} + return file_source_metadata_proto_rawDescGZIP(), []int{21} } func (x *Test) GetFile() string { @@ -2136,7 +2184,7 @@ type Jenkins struct { func (x *Jenkins) Reset() { *x = Jenkins{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[21] + mi := &file_source_metadata_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2149,7 +2197,7 @@ func (x *Jenkins) String() string { func (*Jenkins) ProtoMessage() {} func (x *Jenkins) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[21] + mi := &file_source_metadata_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2162,7 +2210,7 @@ func (x *Jenkins) ProtoReflect() protoreflect.Message { // Deprecated: Use Jenkins.ProtoReflect.Descriptor instead. func (*Jenkins) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{21} + return file_source_metadata_proto_rawDescGZIP(), []int{22} } func (x *Jenkins) GetProjectName() string { @@ -2213,7 +2261,7 @@ type Teams struct { func (x *Teams) Reset() { *x = Teams{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[22] + mi := &file_source_metadata_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2226,7 +2274,7 @@ func (x *Teams) String() string { func (*Teams) ProtoMessage() {} func (x *Teams) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[22] + mi := &file_source_metadata_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2239,7 +2287,7 @@ func (x *Teams) ProtoReflect() protoreflect.Message { // Deprecated: Use Teams.ProtoReflect.Descriptor instead. func (*Teams) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{22} + return file_source_metadata_proto_rawDescGZIP(), []int{23} } func (x *Teams) GetChannelId() string { @@ -2329,7 +2377,7 @@ type Artifactory struct { func (x *Artifactory) Reset() { *x = Artifactory{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[23] + mi := &file_source_metadata_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2342,7 +2390,7 @@ func (x *Artifactory) String() string { func (*Artifactory) ProtoMessage() {} func (x *Artifactory) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[23] + mi := &file_source_metadata_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2355,7 +2403,7 @@ func (x *Artifactory) ProtoReflect() protoreflect.Message { // Deprecated: Use Artifactory.ProtoReflect.Descriptor instead. func (*Artifactory) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{23} + return file_source_metadata_proto_rawDescGZIP(), []int{24} } func (x *Artifactory) GetRepo() string { @@ -2416,7 +2464,7 @@ type Syslog struct { func (x *Syslog) Reset() { *x = Syslog{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[24] + mi := &file_source_metadata_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2429,7 +2477,7 @@ func (x *Syslog) String() string { func (*Syslog) ProtoMessage() {} func (x *Syslog) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[24] + mi := &file_source_metadata_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2442,7 +2490,7 @@ func (x *Syslog) ProtoReflect() protoreflect.Message { // Deprecated: Use Syslog.ProtoReflect.Descriptor instead. func (*Syslog) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{24} + return file_source_metadata_proto_rawDescGZIP(), []int{25} } func (x *Syslog) GetHostname() string { @@ -2503,7 +2551,7 @@ type Forager struct { func (x *Forager) Reset() { *x = Forager{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[25] + mi := &file_source_metadata_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2516,7 +2564,7 @@ func (x *Forager) String() string { func (*Forager) ProtoMessage() {} func (x *Forager) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[25] + mi := &file_source_metadata_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2529,7 +2577,7 @@ func (x *Forager) ProtoReflect() protoreflect.Message { // Deprecated: Use Forager.ProtoReflect.Descriptor instead. func (*Forager) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{25} + return file_source_metadata_proto_rawDescGZIP(), []int{26} } func (m *Forager) GetMetadata() isForager_Metadata { @@ -2587,19 +2635,21 @@ type SharePoint struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Link string `protobuf:"bytes,1,opt,name=link,proto3" json:"link,omitempty"` - Timestamp string `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Author string `protobuf:"bytes,3,opt,name=author,proto3" json:"author,omitempty"` - Title string `protobuf:"bytes,4,opt,name=title,proto3" json:"title,omitempty"` - Views int64 `protobuf:"varint,5,opt,name=views,proto3" json:"views,omitempty"` - Docid string `protobuf:"bytes,6,opt,name=docid,proto3" json:"docid,omitempty"` - Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"` + Link string `protobuf:"bytes,1,opt,name=link,proto3" json:"link,omitempty"` + Timestamp string `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Author string `protobuf:"bytes,3,opt,name=author,proto3" json:"author,omitempty"` + Title string `protobuf:"bytes,4,opt,name=title,proto3" json:"title,omitempty"` + Views int64 `protobuf:"varint,5,opt,name=views,proto3" json:"views,omitempty"` + Docid string `protobuf:"bytes,6,opt,name=docid,proto3" json:"docid,omitempty"` + Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"` + Version int64 `protobuf:"varint,8,opt,name=version,proto3" json:"version,omitempty"` + CommentNumber int64 `protobuf:"varint,9,opt,name=comment_number,json=commentNumber,proto3" json:"comment_number,omitempty"` } func (x *SharePoint) Reset() { *x = SharePoint{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[26] + mi := &file_source_metadata_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2612,7 +2662,7 @@ func (x *SharePoint) String() string { func (*SharePoint) ProtoMessage() {} func (x *SharePoint) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[26] + mi := &file_source_metadata_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2625,7 +2675,7 @@ func (x *SharePoint) ProtoReflect() protoreflect.Message { // Deprecated: Use SharePoint.ProtoReflect.Descriptor instead. func (*SharePoint) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{26} + return file_source_metadata_proto_rawDescGZIP(), []int{27} } func (x *SharePoint) GetLink() string { @@ -2677,6 +2727,20 @@ func (x *SharePoint) GetEmail() string { return "" } +func (x *SharePoint) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *SharePoint) GetCommentNumber() int64 { + if x != nil { + return x.CommentNumber + } + return 0 +} + type GoogleDrive struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2694,7 +2758,7 @@ type GoogleDrive struct { func (x *GoogleDrive) Reset() { *x = GoogleDrive{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[27] + mi := &file_source_metadata_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2707,7 +2771,7 @@ func (x *GoogleDrive) String() string { func (*GoogleDrive) ProtoMessage() {} func (x *GoogleDrive) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[27] + mi := &file_source_metadata_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2720,7 +2784,7 @@ func (x *GoogleDrive) ProtoReflect() protoreflect.Message { // Deprecated: Use GoogleDrive.ProtoReflect.Descriptor instead. func (*GoogleDrive) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{27} + return file_source_metadata_proto_rawDescGZIP(), []int{28} } func (x *GoogleDrive) GetFile() string { @@ -2793,7 +2857,7 @@ type AzureRepos struct { func (x *AzureRepos) Reset() { *x = AzureRepos{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[28] + mi := &file_source_metadata_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2806,7 +2870,7 @@ func (x *AzureRepos) String() string { func (*AzureRepos) ProtoMessage() {} func (x *AzureRepos) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[28] + mi := &file_source_metadata_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2819,7 +2883,7 @@ func (x *AzureRepos) ProtoReflect() protoreflect.Message { // Deprecated: Use AzureRepos.ProtoReflect.Descriptor instead. func (*AzureRepos) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{28} + return file_source_metadata_proto_rawDescGZIP(), []int{29} } func (x *AzureRepos) GetLink() string { @@ -2924,7 +2988,7 @@ type Postman struct { func (x *Postman) Reset() { *x = Postman{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[29] + mi := &file_source_metadata_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2937,7 +3001,7 @@ func (x *Postman) String() string { func (*Postman) ProtoMessage() {} func (x *Postman) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[29] + mi := &file_source_metadata_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2950,7 +3014,7 @@ func (x *Postman) ProtoReflect() protoreflect.Message { // Deprecated: Use Postman.ProtoReflect.Descriptor instead. func (*Postman) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{29} + return file_source_metadata_proto_rawDescGZIP(), []int{30} } func (x *Postman) GetLink() string { @@ -3073,7 +3137,7 @@ type Vector struct { func (x *Vector) Reset() { *x = Vector{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[30] + mi := &file_source_metadata_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3086,7 +3150,7 @@ func (x *Vector) String() string { func (*Vector) ProtoMessage() {} func (x *Vector) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[30] + mi := &file_source_metadata_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3099,7 +3163,7 @@ func (x *Vector) ProtoReflect() protoreflect.Message { // Deprecated: Use Vector.ProtoReflect.Descriptor instead. func (*Vector) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{30} + return file_source_metadata_proto_rawDescGZIP(), []int{31} } func (x *Vector) GetTimestamp() *timestamppb.Timestamp { @@ -3151,7 +3215,7 @@ type Webhook struct { func (x *Webhook) Reset() { *x = Webhook{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[31] + mi := &file_source_metadata_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3164,7 +3228,7 @@ func (x *Webhook) String() string { func (*Webhook) ProtoMessage() {} func (x *Webhook) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[31] + mi := &file_source_metadata_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3177,7 +3241,7 @@ func (x *Webhook) ProtoReflect() protoreflect.Message { // Deprecated: Use Webhook.ProtoReflect.Descriptor instead. func (*Webhook) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{31} + return file_source_metadata_proto_rawDescGZIP(), []int{32} } func (m *Webhook) GetData() isWebhook_Data { @@ -3217,7 +3281,7 @@ type Elasticsearch struct { func (x *Elasticsearch) Reset() { *x = Elasticsearch{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[32] + mi := &file_source_metadata_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3230,7 +3294,7 @@ func (x *Elasticsearch) String() string { func (*Elasticsearch) ProtoMessage() {} func (x *Elasticsearch) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[32] + mi := &file_source_metadata_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3243,7 +3307,7 @@ func (x *Elasticsearch) ProtoReflect() protoreflect.Message { // Deprecated: Use Elasticsearch.ProtoReflect.Descriptor instead. func (*Elasticsearch) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{32} + return file_source_metadata_proto_rawDescGZIP(), []int{33} } func (x *Elasticsearch) GetIndex() string { @@ -3286,7 +3350,7 @@ type Sentry struct { func (x *Sentry) Reset() { *x = Sentry{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[33] + mi := &file_source_metadata_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3299,7 +3363,7 @@ func (x *Sentry) String() string { func (*Sentry) ProtoMessage() {} func (x *Sentry) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[33] + mi := &file_source_metadata_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3312,7 +3376,7 @@ func (x *Sentry) ProtoReflect() protoreflect.Message { // Deprecated: Use Sentry.ProtoReflect.Descriptor instead. func (*Sentry) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{33} + return file_source_metadata_proto_rawDescGZIP(), []int{34} } func (x *Sentry) GetEventId() string { @@ -3387,7 +3451,7 @@ type Stdin struct { func (x *Stdin) Reset() { *x = Stdin{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[34] + mi := &file_source_metadata_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3400,7 +3464,7 @@ func (x *Stdin) String() string { func (*Stdin) ProtoMessage() {} func (x *Stdin) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[34] + mi := &file_source_metadata_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3413,7 +3477,7 @@ func (x *Stdin) ProtoReflect() protoreflect.Message { // Deprecated: Use Stdin.ProtoReflect.Descriptor instead. func (*Stdin) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{34} + return file_source_metadata_proto_rawDescGZIP(), []int{35} } type SlackContinuous struct { @@ -3436,7 +3500,7 @@ type SlackContinuous struct { func (x *SlackContinuous) Reset() { *x = SlackContinuous{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[35] + mi := &file_source_metadata_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3449,7 +3513,7 @@ func (x *SlackContinuous) String() string { func (*SlackContinuous) ProtoMessage() {} func (x *SlackContinuous) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[35] + mi := &file_source_metadata_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3462,7 +3526,7 @@ func (x *SlackContinuous) ProtoReflect() protoreflect.Message { // Deprecated: Use SlackContinuous.ProtoReflect.Descriptor instead. func (*SlackContinuous) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{35} + return file_source_metadata_proto_rawDescGZIP(), []int{36} } func (x *SlackContinuous) GetChannelId() string { @@ -3546,7 +3610,7 @@ type JSONEnumerator struct { func (x *JSONEnumerator) Reset() { *x = JSONEnumerator{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[36] + mi := &file_source_metadata_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3559,7 +3623,7 @@ func (x *JSONEnumerator) String() string { func (*JSONEnumerator) ProtoMessage() {} func (x *JSONEnumerator) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[36] + mi := &file_source_metadata_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3572,7 +3636,7 @@ func (x *JSONEnumerator) ProtoReflect() protoreflect.Message { // Deprecated: Use JSONEnumerator.ProtoReflect.Descriptor instead. func (*JSONEnumerator) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{36} + return file_source_metadata_proto_rawDescGZIP(), []int{37} } func (x *JSONEnumerator) GetMetadata() string { @@ -3625,13 +3689,14 @@ type MetaData struct { // *MetaData_Stdin // *MetaData_SlackContinuous // *MetaData_JsonEnumerator + // *MetaData_AwsSecretsManager Data isMetaData_Data `protobuf_oneof:"data"` } func (x *MetaData) Reset() { *x = MetaData{} if protoimpl.UnsafeEnabled { - mi := &file_source_metadata_proto_msgTypes[37] + mi := &file_source_metadata_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3644,7 +3709,7 @@ func (x *MetaData) String() string { func (*MetaData) ProtoMessage() {} func (x *MetaData) ProtoReflect() protoreflect.Message { - mi := &file_source_metadata_proto_msgTypes[37] + mi := &file_source_metadata_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3657,7 +3722,7 @@ func (x *MetaData) ProtoReflect() protoreflect.Message { // Deprecated: Use MetaData.ProtoReflect.Descriptor instead. func (*MetaData) Descriptor() ([]byte, []int) { - return file_source_metadata_proto_rawDescGZIP(), []int{37} + return file_source_metadata_proto_rawDescGZIP(), []int{38} } func (m *MetaData) GetData() isMetaData_Data { @@ -3919,6 +3984,13 @@ func (x *MetaData) GetJsonEnumerator() *JSONEnumerator { return nil } +func (x *MetaData) GetAwsSecretsManager() *AWSSecretsManager { + if x, ok := x.GetData().(*MetaData_AwsSecretsManager); ok { + return x.AwsSecretsManager + } + return nil +} + type isMetaData_Data interface { isMetaData_Data() } @@ -4067,6 +4139,10 @@ type MetaData_JsonEnumerator struct { JsonEnumerator *JSONEnumerator `protobuf:"bytes,36,opt,name=jsonEnumerator,proto3,oneof"` } +type MetaData_AwsSecretsManager struct { + AwsSecretsManager *AWSSecretsManager `protobuf:"bytes,37,opt,name=awsSecretsManager,proto3,oneof"` +} + func (*MetaData_Azure) isMetaData_Data() {} func (*MetaData_Bitbucket) isMetaData_Data() {} @@ -4139,6 +4215,8 @@ func (*MetaData_SlackContinuous) isMetaData_Data() {} func (*MetaData_JsonEnumerator) isMetaData_Data() {} +func (*MetaData_AwsSecretsManager) isMetaData_Data() {} + var File_source_metadata_proto protoreflect.FileDescriptor var file_source_metadata_proto_rawDesc = []byte{ @@ -4146,118 +4224,182 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7f, 0x0a, 0x05, 0x41, 0x7a, 0x75, - 0x72, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x86, 0x02, 0x0a, 0x09, 0x42, - 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, - 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x6e, - 0x69, 0x70, 0x70, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x69, 0x74, - 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, - 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, - 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, - 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, - 0x69, 0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6f, 0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, - 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x0b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xcd, 0x01, 0x0a, 0x08, 0x43, - 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x12, 0x19, 0x0a, 0x08, 0x76, 0x63, 0x73, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x63, 0x73, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x77, 0x0a, 0x11, 0x41, 0x57, 0x53, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, + 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x62, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x5f, 0x61, 0x72, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x43, 0xfa, 0x42, 0x40, + 0x72, 0x3e, 0x32, 0x3c, 0x5e, 0x61, 0x72, 0x6e, 0x3a, 0x61, 0x77, 0x73, 0x5b, 0x5e, 0x3a, 0x5d, + 0x2a, 0x3a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, + 0x3a, 0x5b, 0x61, 0x2d, 0x7a, 0x30, 0x2d, 0x39, 0x2d, 0x5d, 0x2b, 0x3a, 0x5b, 0x30, 0x2d, 0x39, + 0x5d, 0x7b, 0x31, 0x32, 0x7d, 0x3a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x3a, 0x2e, 0x2b, 0x24, + 0x52, 0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x41, 0x72, 0x6e, 0x22, 0x7f, 0x0a, 0x05, 0x41, + 0x7a, 0x75, 0x72, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, + 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, + 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x86, 0x02, 0x0a, + 0x09, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1e, + 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, + 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, + 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, + 0x69, 0x74, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x69, 0x74, 0x6c, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, + 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, + 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, + 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, + 0x69, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xcd, 0x01, 0x0a, + 0x08, 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x12, 0x19, 0x0a, 0x08, 0x76, 0x63, 0x73, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x63, 0x73, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, + 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x73, 0x74, 0x65, + 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, + 0x65, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xb4, 0x01, 0x0a, + 0x08, 0x54, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, + 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, + 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6e, + 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x6f, 0x62, 0x5f, + 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x6f, + 0x62, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x70, + 0x75, 0x62, 0x6c, 0x69, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x75, 0x62, + 0x6c, 0x69, 0x63, 0x22, 0xfb, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, + 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x70, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, + 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, + 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, + 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x49, + 0x64, 0x22, 0x5a, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x74, + 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x22, 0xa3, 0x01, + 0x0a, 0x03, 0x45, 0x43, 0x52, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x79, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x12, + 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, + 0x79, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, + 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, + 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, + 0x61, 0x69, 0x6c, 0x22, 0x5e, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, + 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, + 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, + 0x69, 0x6e, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x03, 0x47, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1e, 0x0a, + 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, + 0x69, 0x6e, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, + 0x32, 0x0a, 0x15, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, + 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, + 0x61, 0x74, 0x68, 0x22, 0xbd, 0x02, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, + 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, + 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x21, - 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x65, 0x70, - 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xb4, 0x01, 0x0a, 0x08, 0x54, - 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, - 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, - 0x6f, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6e, 0x75, 0x6d, - 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x6f, 0x62, 0x4e, - 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, - 0x6c, 0x69, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, - 0x63, 0x22, 0xfb, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x70, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, - 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x75, 0x73, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, - 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, - 0x5a, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x22, 0xa3, 0x01, 0x0a, 0x03, - 0x45, 0x43, 0x52, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x79, 0x65, 0x72, 0x12, 0x14, 0x0a, - 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x12, - 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, - 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, - 0x6c, 0x22, 0x5e, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, - 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, - 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, - 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x03, 0x47, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, - 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, - 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x32, 0x0a, + 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, + 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, + 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, + 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, + 0x6e, 0x65, 0x12, 0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, + 0x32, 0x0a, 0x15, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, + 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, + 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, + 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, + 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, + 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, + 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x5f, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x32, 0x0a, 0x15, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x72, 0x65, + 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, - 0x68, 0x22, 0xbd, 0x02, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, + 0x68, 0x22, 0xd8, 0x01, 0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, + 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x63, 0x6c, 0x73, 0x18, 0x07, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x61, 0x63, 0x6c, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xb3, 0x02, 0x0a, + 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, @@ -4273,493 +4415,447 @@ var file_source_metadata_proto_rawDesc = []byte{ 0x12, 0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, - 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x32, 0x0a, - 0x15, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x72, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, - 0x68, 0x22, 0xcb, 0x02, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, - 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, - 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, - 0x69, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, - 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, - 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x5f, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x32, 0x0a, 0x15, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, - 0x70, 0x61, 0x74, 0x68, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x72, 0x65, 0x70, 0x6f, - 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x22, - 0xd8, 0x01, 0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, - 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, + 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x23, 0x0a, + 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, + 0x70, 0x65, 0x22, 0x98, 0x01, 0x0a, 0x04, 0x4a, 0x69, 0x72, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x69, + 0x73, 0x73, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x73, 0x73, 0x75, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, + 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, + 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, + 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, + 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x77, 0x0a, + 0x03, 0x4e, 0x50, 0x4d, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, + 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x78, 0x0a, 0x04, 0x50, 0x79, 0x50, 0x69, 0x12, 0x12, + 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, + 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, + 0x22, 0x78, 0x0a, 0x02, 0x53, 0x33, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1c, 0x0a, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x97, 0x02, 0x0a, 0x05, 0x53, + 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, + 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, + 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x3b, 0x0a, 0x0a, 0x76, + 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0a, 0x76, 0x69, + 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x96, 0x01, 0x0a, 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, + 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, + 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a, + 0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65, + 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, - 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, - 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x64, 0x41, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, - 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x64, 0x41, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x63, 0x6c, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x04, 0x61, 0x63, 0x6c, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xb3, 0x02, 0x0a, 0x0b, 0x48, - 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, - 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, - 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, - 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, - 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, - 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x3b, - 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, - 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x72, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x22, 0x98, 0x01, 0x0a, 0x04, 0x4a, 0x69, 0x72, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x73, 0x73, - 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x73, 0x73, 0x75, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x6c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, - 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1c, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x77, 0x0a, 0x03, 0x4e, - 0x50, 0x4d, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, - 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, - 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, - 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x78, 0x0a, 0x04, 0x50, 0x79, 0x50, 0x69, 0x12, 0x12, 0x0a, 0x04, - 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x6c, - 0x65, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, - 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0x78, - 0x0a, 0x02, 0x53, 0x33, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x97, 0x02, 0x0a, 0x05, 0x53, 0x6c, 0x61, - 0x63, 0x6b, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, - 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6c, - 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, - 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, - 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, - 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x96, 0x01, 0x0a, 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, - 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, - 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a, 0x04, 0x54, - 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e, 0x6b, - 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, - 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x62, 0x75, - 0x69, 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, - 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x90, 0x02, 0x0a, 0x05, - 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, - 0x65, 0x6c, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, - 0x6e, 0x65, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, - 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1a, 0x0a, 0x08, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x61, 0x6d, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x61, - 0x6d, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x22, 0x99, - 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, - 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, - 0x70, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xa8, 0x01, 0x0a, 0x06, 0x53, - 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x70, 0x70, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, - 0x72, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x6f, - 0x63, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x61, 0x63, - 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x61, 0x63, - 0x69, 0x6c, 0x69, 0x74, 0x79, 0x22, 0x9f, 0x01, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x72, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x90, 0x02, + 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, + 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1a, + 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, + 0x61, 0x6d, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, + 0x65, 0x61, 0x6d, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x65, 0x61, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x64, + 0x22, 0x99, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, + 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x72, 0x65, 0x70, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, + 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, + 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xa8, 0x01, 0x0a, + 0x06, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x70, 0x70, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x70, 0x72, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, + 0x72, 0x6f, 0x63, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, + 0x61, 0x63, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, + 0x61, 0x63, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x22, 0x9f, 0x01, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x72, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, + 0x12, 0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, + 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x42, 0x0a, 0x0a, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xef, 0x01, 0x0a, 0x0a, 0x53, 0x68, + 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x75, 0x74, 0x68, + 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x69, 0x65, 0x77, + 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x64, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64, + 0x6f, 0x63, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x5f, + 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x63, 0x6f, + 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0xbf, 0x01, 0x0a, 0x0b, + 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, + 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x12, + 0x28, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, + 0x5f, 0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4d, + 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, + 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, + 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, + 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, + 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, + 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, + 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, + 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, + 0x12, 0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, + 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, + 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, + 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xf4, 0x04, 0x0a, 0x07, + 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x25, 0x0a, 0x0e, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x55, 0x75, + 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x27, + 0x0a, 0x0f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x29, + 0x0a, 0x10, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, + 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x6f, 0x6c, 0x64, + 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, + 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x49, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x24, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, + 0x04, 0x08, 0x0e, 0x10, 0x0f, 0x4a, 0x04, 0x08, 0x0f, 0x10, 0x10, 0x52, 0x0a, 0x67, 0x6c, 0x6f, + 0x62, 0x61, 0x6c, 0x73, 0x5f, 0x69, 0x64, 0x52, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x52, 0x0d, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x22, 0xa5, 0x01, 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x38, 0x0a, + 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, + 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x22, 0x44, 0x0a, 0x07, 0x57, 0x65, + 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x12, 0x31, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x48, 0x00, + 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x22, 0x64, 0x0a, 0x0d, 0x45, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, + 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6f, + 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xc9, 0x02, 0x0a, 0x06, 0x53, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0f, + 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x6c, 0x75, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x10, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6c, + 0x75, 0x67, 0x12, 0x3a, 0x0a, 0x19, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x6c, 0x75, 0x67, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x6c, 0x75, 0x67, + 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x73, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x69, 0x73, 0x73, 0x75, 0x65, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64, + 0x61, 0x74, 0x65, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, + 0x6e, 0x6b, 0x22, 0x07, 0x0a, 0x05, 0x53, 0x74, 0x64, 0x69, 0x6e, 0x22, 0xc4, 0x02, 0x0a, 0x0f, + 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x12, + 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, + 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, + 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x21, 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x49, 0x64, 0x22, 0x2c, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, + 0x61, 0x74, 0x6f, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x22, 0x93, 0x10, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, + 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, + 0x7a, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, + 0x09, 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, + 0x62, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, + 0x63, 0x6c, 0x65, 0x63, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, + 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x48, 0x00, 0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, + 0x63, 0x69, 0x12, 0x3d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, + 0x65, 0x12, 0x31, 0x0a, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x61, 0x74, 0x61, 0x2e, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, + 0x63, 0x6b, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12, 0x2b, - 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, - 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x42, 0x0a, 0x0a, 0x08, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xae, 0x01, 0x0a, 0x0a, 0x53, 0x68, 0x61, 0x72, - 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x75, 0x74, 0x68, - 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x12, 0x14, 0x0a, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x69, 0x65, 0x77, 0x73, 0x12, 0x14, 0x0a, 0x05, - 0x64, 0x6f, 0x63, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64, 0x6f, 0x63, - 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x22, 0xbf, 0x01, 0x0a, 0x0b, 0x47, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, - 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x12, 0x28, 0x0a, 0x10, - 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x62, 0x79, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, - 0x66, 0x69, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xcb, 0x02, 0x0a, 0x0a, 0x41, - 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, - 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, - 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, - 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x3b, 0x0a, - 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0a, - 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x72, 0x67, 0x61, - 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xf4, 0x04, 0x0a, 0x07, 0x50, 0x6f, 0x73, - 0x74, 0x6d, 0x61, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x6f, 0x72, 0x6b, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x55, 0x75, 0x69, 0x64, 0x12, - 0x25, 0x0a, 0x0e, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x6f, 0x6c, 0x64, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x6f, 0x6c, - 0x64, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x6f, 0x6c, 0x64, - 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x49, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, - 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x0c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x49, - 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x0e, - 0x10, 0x0f, 0x4a, 0x04, 0x08, 0x0f, 0x10, 0x10, 0x52, 0x0a, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, - 0x73, 0x5f, 0x69, 0x64, 0x52, 0x0a, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x52, 0x0d, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, - 0xa5, 0x01, 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, 0x6f, 0x63, 0x61, - 0x74, 0x6f, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x22, 0x44, 0x0a, 0x07, 0x57, 0x65, 0x62, 0x68, 0x6f, - 0x6f, 0x6b, 0x12, 0x31, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x06, 0x76, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x64, 0x0a, - 0x0d, 0x45, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x14, - 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x22, 0xc9, 0x02, 0x0a, 0x06, 0x53, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x19, - 0x0a, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x6f, 0x72, 0x67, - 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x73, 0x6c, 0x75, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6f, - 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x6c, 0x75, 0x67, 0x12, - 0x3a, 0x0a, 0x19, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x64, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x17, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x44, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x6c, 0x75, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x6c, 0x75, 0x67, 0x12, 0x19, 0x0a, - 0x08, 0x69, 0x73, 0x73, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x69, 0x73, 0x73, 0x75, 0x65, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x61, 0x74, 0x65, - 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x64, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6c, - 0x69, 0x6e, 0x6b, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x22, - 0x07, 0x0a, 0x05, 0x53, 0x74, 0x64, 0x69, 0x6e, 0x22, 0xc4, 0x02, 0x0a, 0x0f, 0x53, 0x6c, 0x61, - 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x12, 0x1d, 0x0a, 0x0a, - 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, - 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x17, 0x0a, 0x07, - 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, - 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, - 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, - 0x61, 0x69, 0x6c, 0x12, 0x3b, 0x0a, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, - 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0a, 0x76, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, - 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, - 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x22, - 0x2c, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, - 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, 0x0f, - 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a, - 0x75, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, - 0x65, 0x48, 0x00, 0x52, 0x05, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x12, 0x3a, 0x0a, 0x09, 0x62, 0x69, - 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x61, 0x74, 0x61, 0x2e, 0x45, 0x43, 0x52, 0x48, 0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, + 0x0a, 0x03, 0x67, 0x63, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, + 0x53, 0x48, 0x00, 0x52, 0x03, 0x67, 0x63, 0x73, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, + 0x69, 0x74, 0x6c, 0x61, 0x62, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, + 0x74, 0x6c, 0x61, 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, + 0x0a, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, + 0x69, 0x72, 0x61, 0x48, 0x00, 0x52, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, + 0x70, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, + 0x52, 0x03, 0x6e, 0x70, 0x6d, 0x12, 0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, + 0x70, 0x69, 0x12, 0x25, 0x0a, 0x02, 0x73, 0x33, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, + 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x53, 0x33, 0x48, 0x00, 0x52, 0x02, 0x73, 0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, + 0x63, 0x6b, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, + 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, + 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x62, 0x69, 0x74, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, - 0x63, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x6c, - 0x65, 0x43, 0x49, 0x48, 0x00, 0x52, 0x08, 0x63, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x63, 0x69, 0x12, - 0x3d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, + 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, + 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, + 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, + 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x65, 0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, + 0x3a, 0x0a, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, + 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, + 0x65, 0x72, 0x72, 0x69, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x72, 0x72, 0x69, 0x74, 0x48, 0x00, 0x52, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, + 0x0a, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, + 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, + 0x65, 0x61, 0x6d, 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x31, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, + 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, + 0x00, 0x52, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x34, 0x0a, 0x07, 0x66, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x72, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x72, 0x48, 0x00, 0x52, 0x07, 0x66, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x72, 0x12, + 0x3d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, - 0x48, 0x00, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x31, - 0x0a, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, + 0x48, 0x00, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x40, + 0x0a, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x18, 0x1a, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, + 0x65, 0x48, 0x00, 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, + 0x12, 0x3d, 0x0a, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x1b, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, + 0x73, 0x48, 0x00, 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, + 0x37, 0x0a, 0x08, 0x74, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x18, 0x1c, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x19, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x48, 0x00, 0x52, 0x08, + 0x74, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x12, 0x34, 0x0a, 0x07, 0x70, 0x6f, 0x73, 0x74, + 0x6d, 0x61, 0x6e, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x6f, 0x73, 0x74, + 0x6d, 0x61, 0x6e, 0x48, 0x00, 0x52, 0x07, 0x70, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x12, 0x34, + 0x0a, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x57, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x48, 0x00, 0x52, 0x07, 0x77, 0x65, 0x62, + 0x68, 0x6f, 0x6f, 0x6b, 0x12, 0x46, 0x0a, 0x0d, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x6c, + 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0d, 0x65, + 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x40, 0x0a, 0x0b, + 0x68, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x18, 0x20, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x48, + 0x00, 0x52, 0x0b, 0x68, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x12, 0x31, + 0x0a, 0x06, 0x73, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, - 0x72, 0x12, 0x28, 0x0a, 0x03, 0x65, 0x63, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x45, 0x43, 0x52, 0x48, 0x00, 0x52, 0x03, 0x65, 0x63, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x67, - 0x63, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x43, 0x53, 0x48, 0x00, - 0x52, 0x03, 0x67, 0x63, 0x73, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x48, 0x00, - 0x52, 0x06, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x69, 0x74, 0x6c, - 0x61, 0x62, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x6c, 0x61, - 0x62, 0x48, 0x00, 0x52, 0x06, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x2b, 0x0a, 0x04, 0x6a, - 0x69, 0x72, 0x61, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x69, 0x72, 0x61, - 0x48, 0x00, 0x52, 0x04, 0x6a, 0x69, 0x72, 0x61, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x70, 0x6d, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x50, 0x4d, 0x48, 0x00, 0x52, 0x03, 0x6e, - 0x70, 0x6d, 0x12, 0x2b, 0x0a, 0x04, 0x70, 0x79, 0x70, 0x69, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x50, 0x79, 0x50, 0x69, 0x48, 0x00, 0x52, 0x04, 0x70, 0x79, 0x70, 0x69, 0x12, - 0x25, 0x0a, 0x02, 0x73, 0x33, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x33, - 0x48, 0x00, 0x52, 0x02, 0x73, 0x33, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x18, - 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x48, 0x00, 0x52, - 0x05, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x3d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x28, 0x0a, 0x03, 0x67, 0x69, 0x74, 0x18, 0x10, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x48, 0x00, 0x52, 0x03, 0x67, 0x69, 0x74, 0x12, - 0x2b, 0x0a, 0x04, 0x74, 0x65, 0x73, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x54, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x04, 0x74, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x09, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x48, 0x00, 0x52, 0x09, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x67, 0x65, 0x72, 0x72, - 0x69, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x72, 0x72, 0x69, - 0x74, 0x48, 0x00, 0x52, 0x06, 0x67, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6a, - 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, - 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, - 0x73, 0x12, 0x2e, 0x0a, 0x05, 0x74, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, + 0x2e, 0x53, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x48, 0x00, 0x52, 0x06, 0x73, 0x65, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x05, 0x74, 0x65, 0x61, 0x6d, - 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, - 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x12, 0x31, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x18, 0x17, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x48, 0x00, 0x52, 0x06, - 0x73, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x34, 0x0a, 0x07, 0x66, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x72, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x72, 0x48, 0x00, 0x52, 0x07, 0x66, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x72, 0x12, 0x3d, 0x0a, 0x0a, - 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x65, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x48, 0x00, 0x52, - 0x0a, 0x73, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0b, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x48, 0x00, - 0x52, 0x0b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x3d, 0x0a, - 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x1b, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x48, 0x00, - 0x52, 0x0a, 0x61, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x37, 0x0a, 0x08, - 0x74, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x54, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x48, 0x00, 0x52, 0x08, 0x74, 0x72, 0x61, - 0x76, 0x69, 0x73, 0x43, 0x49, 0x12, 0x34, 0x0a, 0x07, 0x70, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, - 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, - 0x48, 0x00, 0x52, 0x07, 0x70, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x12, 0x34, 0x0a, 0x07, 0x77, - 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, - 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x48, 0x00, 0x52, 0x07, 0x77, 0x65, 0x62, 0x68, 0x6f, 0x6f, - 0x6b, 0x12, 0x46, 0x0a, 0x0d, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x6c, 0x61, 0x73, 0x74, - 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0d, 0x65, 0x6c, 0x61, 0x73, - 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x40, 0x0a, 0x0b, 0x68, 0x75, 0x67, - 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x18, 0x20, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0b, - 0x68, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x06, 0x73, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x65, - 0x6e, 0x74, 0x72, 0x79, 0x48, 0x00, 0x52, 0x06, 0x73, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x2e, - 0x0a, 0x05, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x53, 0x74, 0x64, 0x69, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x12, 0x4c, - 0x0a, 0x0f, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, - 0x73, 0x18, 0x23, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x43, - 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x48, 0x00, 0x52, 0x0f, 0x73, 0x6c, 0x61, - 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x12, 0x49, 0x0a, 0x0e, - 0x6a, 0x73, 0x6f, 0x6e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x24, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, - 0x72, 0x61, 0x74, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x0e, 0x6a, 0x73, 0x6f, 0x6e, 0x45, 0x6e, 0x75, - 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, - 0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, - 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, - 0x76, 0x61, 0x74, 0x65, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, - 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x2a, - 0xc2, 0x03, 0x0a, 0x13, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, - 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x50, 0x41, - 0x52, 0x41, 0x4d, 0x45, 0x54, 0x45, 0x52, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x51, - 0x55, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, - 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x03, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, - 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x44, 0x41, - 0x54, 0x41, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, - 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x52, 0x41, 0x57, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, - 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x55, 0x52, 0x4c, 0x5f, 0x45, - 0x4e, 0x43, 0x4f, 0x44, 0x45, 0x44, 0x10, 0x06, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, - 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x47, 0x52, 0x41, 0x50, 0x48, 0x51, 0x4c, - 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x43, - 0x52, 0x49, 0x50, 0x54, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, - 0x54, 0x5f, 0x55, 0x52, 0x4c, 0x10, 0x09, 0x12, 0x18, 0x0a, 0x14, 0x45, 0x4e, 0x56, 0x49, 0x52, - 0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, - 0x0a, 0x12, 0x18, 0x0a, 0x14, 0x46, 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x41, 0x55, 0x54, 0x48, - 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x46, - 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0c, 0x12, 0x15, - 0x0a, 0x11, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x43, 0x52, - 0x49, 0x50, 0x54, 0x10, 0x0d, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, 0x1c, - 0x0a, 0x18, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, - 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d, - 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x10, 0x10, 0x12, - 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, - 0x45, 0x52, 0x10, 0x11, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x74, 0x61, 0x2e, 0x53, 0x74, 0x64, 0x69, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x73, 0x74, 0x64, 0x69, + 0x6e, 0x12, 0x4c, 0x0a, 0x0f, 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, + 0x75, 0x6f, 0x75, 0x73, 0x18, 0x23, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x6c, 0x61, + 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x48, 0x00, 0x52, 0x0f, + 0x73, 0x6c, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x12, + 0x49, 0x0a, 0x0e, 0x6a, 0x73, 0x6f, 0x6e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, + 0x72, 0x18, 0x24, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, + 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x0e, 0x6a, 0x73, 0x6f, 0x6e, + 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x52, 0x0a, 0x11, 0x61, 0x77, + 0x73, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, + 0x25, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x48, 0x00, 0x52, 0x11, 0x61, 0x77, 0x73, + 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x42, 0x06, + 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x10, 0x00, + 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10, 0x01, 0x12, 0x0a, 0x0a, + 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x75, 0x6e, 0x6b, + 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x2a, 0xc2, 0x03, 0x0a, 0x13, 0x50, 0x6f, 0x73, 0x74, 0x6d, + 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x13, + 0x0a, 0x0f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, + 0x4e, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x51, + 0x55, 0x45, 0x52, 0x59, 0x5f, 0x50, 0x41, 0x52, 0x41, 0x4d, 0x45, 0x54, 0x45, 0x52, 0x10, 0x01, + 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x55, 0x54, 0x48, + 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x52, + 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x03, 0x12, + 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, + 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x52, + 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x52, 0x41, 0x57, 0x10, + 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, + 0x59, 0x5f, 0x55, 0x52, 0x4c, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x45, 0x44, 0x10, 0x06, 0x12, + 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, + 0x47, 0x52, 0x41, 0x50, 0x48, 0x51, 0x4c, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, + 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x08, 0x12, 0x0f, 0x0a, + 0x0b, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x55, 0x52, 0x4c, 0x10, 0x09, 0x12, 0x18, + 0x0a, 0x14, 0x45, 0x4e, 0x56, 0x49, 0x52, 0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x56, 0x41, + 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0a, 0x12, 0x18, 0x0a, 0x14, 0x46, 0x4f, 0x4c, 0x44, + 0x45, 0x52, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, + 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x46, 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x53, 0x43, 0x52, + 0x49, 0x50, 0x54, 0x10, 0x0c, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0d, 0x12, 0x17, 0x0a, 0x13, + 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, + 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, 0x1c, 0x0a, 0x18, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, + 0x4e, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, + 0x42, 0x4f, 0x44, 0x59, 0x10, 0x10, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, + 0x53, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x11, 0x42, 0x43, 0x5a, 0x41, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, + 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, + 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4775,103 +4871,105 @@ func file_source_metadata_proto_rawDescGZIP() []byte { } var file_source_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 38) +var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 39) var file_source_metadata_proto_goTypes = []interface{}{ (Visibility)(0), // 0: source_metadata.Visibility (PostmanLocationType)(0), // 1: source_metadata.PostmanLocationType - (*Azure)(nil), // 2: source_metadata.Azure - (*Bitbucket)(nil), // 3: source_metadata.Bitbucket - (*Buildkite)(nil), // 4: source_metadata.Buildkite - (*CircleCI)(nil), // 5: source_metadata.CircleCI - (*TravisCI)(nil), // 6: source_metadata.TravisCI - (*Confluence)(nil), // 7: source_metadata.Confluence - (*Docker)(nil), // 8: source_metadata.Docker - (*ECR)(nil), // 9: source_metadata.ECR - (*Filesystem)(nil), // 10: source_metadata.Filesystem - (*Git)(nil), // 11: source_metadata.Git - (*Github)(nil), // 12: source_metadata.Github - (*Gitlab)(nil), // 13: source_metadata.Gitlab - (*GCS)(nil), // 14: source_metadata.GCS - (*Huggingface)(nil), // 15: source_metadata.Huggingface - (*Jira)(nil), // 16: source_metadata.Jira - (*NPM)(nil), // 17: source_metadata.NPM - (*PyPi)(nil), // 18: source_metadata.PyPi - (*S3)(nil), // 19: source_metadata.S3 - (*Slack)(nil), // 20: source_metadata.Slack - (*Gerrit)(nil), // 21: source_metadata.Gerrit - (*Test)(nil), // 22: source_metadata.Test - (*Jenkins)(nil), // 23: source_metadata.Jenkins - (*Teams)(nil), // 24: source_metadata.Teams - (*Artifactory)(nil), // 25: source_metadata.Artifactory - (*Syslog)(nil), // 26: source_metadata.Syslog - (*Forager)(nil), // 27: source_metadata.Forager - (*SharePoint)(nil), // 28: source_metadata.SharePoint - (*GoogleDrive)(nil), // 29: source_metadata.GoogleDrive - (*AzureRepos)(nil), // 30: source_metadata.AzureRepos - (*Postman)(nil), // 31: source_metadata.Postman - (*Vector)(nil), // 32: source_metadata.Vector - (*Webhook)(nil), // 33: source_metadata.Webhook - (*Elasticsearch)(nil), // 34: source_metadata.Elasticsearch - (*Sentry)(nil), // 35: source_metadata.Sentry - (*Stdin)(nil), // 36: source_metadata.Stdin - (*SlackContinuous)(nil), // 37: source_metadata.SlackContinuous - (*JSONEnumerator)(nil), // 38: source_metadata.JSONEnumerator - (*MetaData)(nil), // 39: source_metadata.MetaData - (*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp + (*AWSSecretsManager)(nil), // 2: source_metadata.AWSSecretsManager + (*Azure)(nil), // 3: source_metadata.Azure + (*Bitbucket)(nil), // 4: source_metadata.Bitbucket + (*Buildkite)(nil), // 5: source_metadata.Buildkite + (*CircleCI)(nil), // 6: source_metadata.CircleCI + (*TravisCI)(nil), // 7: source_metadata.TravisCI + (*Confluence)(nil), // 8: source_metadata.Confluence + (*Docker)(nil), // 9: source_metadata.Docker + (*ECR)(nil), // 10: source_metadata.ECR + (*Filesystem)(nil), // 11: source_metadata.Filesystem + (*Git)(nil), // 12: source_metadata.Git + (*Github)(nil), // 13: source_metadata.Github + (*Gitlab)(nil), // 14: source_metadata.Gitlab + (*GCS)(nil), // 15: source_metadata.GCS + (*Huggingface)(nil), // 16: source_metadata.Huggingface + (*Jira)(nil), // 17: source_metadata.Jira + (*NPM)(nil), // 18: source_metadata.NPM + (*PyPi)(nil), // 19: source_metadata.PyPi + (*S3)(nil), // 20: source_metadata.S3 + (*Slack)(nil), // 21: source_metadata.Slack + (*Gerrit)(nil), // 22: source_metadata.Gerrit + (*Test)(nil), // 23: source_metadata.Test + (*Jenkins)(nil), // 24: source_metadata.Jenkins + (*Teams)(nil), // 25: source_metadata.Teams + (*Artifactory)(nil), // 26: source_metadata.Artifactory + (*Syslog)(nil), // 27: source_metadata.Syslog + (*Forager)(nil), // 28: source_metadata.Forager + (*SharePoint)(nil), // 29: source_metadata.SharePoint + (*GoogleDrive)(nil), // 30: source_metadata.GoogleDrive + (*AzureRepos)(nil), // 31: source_metadata.AzureRepos + (*Postman)(nil), // 32: source_metadata.Postman + (*Vector)(nil), // 33: source_metadata.Vector + (*Webhook)(nil), // 34: source_metadata.Webhook + (*Elasticsearch)(nil), // 35: source_metadata.Elasticsearch + (*Sentry)(nil), // 36: source_metadata.Sentry + (*Stdin)(nil), // 37: source_metadata.Stdin + (*SlackContinuous)(nil), // 38: source_metadata.SlackContinuous + (*JSONEnumerator)(nil), // 39: source_metadata.JSONEnumerator + (*MetaData)(nil), // 40: source_metadata.MetaData + (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp } var file_source_metadata_proto_depIdxs = []int32{ 0, // 0: source_metadata.Github.visibility:type_name -> source_metadata.Visibility 0, // 1: source_metadata.Huggingface.visibility:type_name -> source_metadata.Visibility 0, // 2: source_metadata.Slack.visibility:type_name -> source_metadata.Visibility - 12, // 3: source_metadata.Forager.github:type_name -> source_metadata.Github - 17, // 4: source_metadata.Forager.npm:type_name -> source_metadata.NPM - 18, // 5: source_metadata.Forager.pypi:type_name -> source_metadata.PyPi + 13, // 3: source_metadata.Forager.github:type_name -> source_metadata.Github + 18, // 4: source_metadata.Forager.npm:type_name -> source_metadata.NPM + 19, // 5: source_metadata.Forager.pypi:type_name -> source_metadata.PyPi 0, // 6: source_metadata.AzureRepos.visibility:type_name -> source_metadata.Visibility 1, // 7: source_metadata.Postman.location_type:type_name -> source_metadata.PostmanLocationType - 40, // 8: source_metadata.Vector.timestamp:type_name -> google.protobuf.Timestamp - 32, // 9: source_metadata.Webhook.vector:type_name -> source_metadata.Vector + 41, // 8: source_metadata.Vector.timestamp:type_name -> google.protobuf.Timestamp + 33, // 9: source_metadata.Webhook.vector:type_name -> source_metadata.Vector 0, // 10: source_metadata.SlackContinuous.visibility:type_name -> source_metadata.Visibility - 2, // 11: source_metadata.MetaData.azure:type_name -> source_metadata.Azure - 3, // 12: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket - 5, // 13: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI - 7, // 14: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence - 8, // 15: source_metadata.MetaData.docker:type_name -> source_metadata.Docker - 9, // 16: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR - 14, // 17: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS - 12, // 18: source_metadata.MetaData.github:type_name -> source_metadata.Github - 13, // 19: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab - 16, // 20: source_metadata.MetaData.jira:type_name -> source_metadata.Jira - 17, // 21: source_metadata.MetaData.npm:type_name -> source_metadata.NPM - 18, // 22: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi - 19, // 23: source_metadata.MetaData.s3:type_name -> source_metadata.S3 - 20, // 24: source_metadata.MetaData.slack:type_name -> source_metadata.Slack - 10, // 25: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem - 11, // 26: source_metadata.MetaData.git:type_name -> source_metadata.Git - 22, // 27: source_metadata.MetaData.test:type_name -> source_metadata.Test - 4, // 28: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite - 21, // 29: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit - 23, // 30: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins - 24, // 31: source_metadata.MetaData.teams:type_name -> source_metadata.Teams - 25, // 32: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory - 26, // 33: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog - 27, // 34: source_metadata.MetaData.forager:type_name -> source_metadata.Forager - 28, // 35: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint - 29, // 36: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive - 30, // 37: source_metadata.MetaData.azureRepos:type_name -> source_metadata.AzureRepos - 6, // 38: source_metadata.MetaData.travisCI:type_name -> source_metadata.TravisCI - 31, // 39: source_metadata.MetaData.postman:type_name -> source_metadata.Postman - 33, // 40: source_metadata.MetaData.webhook:type_name -> source_metadata.Webhook - 34, // 41: source_metadata.MetaData.elasticsearch:type_name -> source_metadata.Elasticsearch - 15, // 42: source_metadata.MetaData.huggingface:type_name -> source_metadata.Huggingface - 35, // 43: source_metadata.MetaData.sentry:type_name -> source_metadata.Sentry - 36, // 44: source_metadata.MetaData.stdin:type_name -> source_metadata.Stdin - 37, // 45: source_metadata.MetaData.slackContinuous:type_name -> source_metadata.SlackContinuous - 38, // 46: source_metadata.MetaData.jsonEnumerator:type_name -> source_metadata.JSONEnumerator - 47, // [47:47] is the sub-list for method output_type - 47, // [47:47] is the sub-list for method input_type - 47, // [47:47] is the sub-list for extension type_name - 47, // [47:47] is the sub-list for extension extendee - 0, // [0:47] is the sub-list for field type_name + 3, // 11: source_metadata.MetaData.azure:type_name -> source_metadata.Azure + 4, // 12: source_metadata.MetaData.bitbucket:type_name -> source_metadata.Bitbucket + 6, // 13: source_metadata.MetaData.circleci:type_name -> source_metadata.CircleCI + 8, // 14: source_metadata.MetaData.confluence:type_name -> source_metadata.Confluence + 9, // 15: source_metadata.MetaData.docker:type_name -> source_metadata.Docker + 10, // 16: source_metadata.MetaData.ecr:type_name -> source_metadata.ECR + 15, // 17: source_metadata.MetaData.gcs:type_name -> source_metadata.GCS + 13, // 18: source_metadata.MetaData.github:type_name -> source_metadata.Github + 14, // 19: source_metadata.MetaData.gitlab:type_name -> source_metadata.Gitlab + 17, // 20: source_metadata.MetaData.jira:type_name -> source_metadata.Jira + 18, // 21: source_metadata.MetaData.npm:type_name -> source_metadata.NPM + 19, // 22: source_metadata.MetaData.pypi:type_name -> source_metadata.PyPi + 20, // 23: source_metadata.MetaData.s3:type_name -> source_metadata.S3 + 21, // 24: source_metadata.MetaData.slack:type_name -> source_metadata.Slack + 11, // 25: source_metadata.MetaData.filesystem:type_name -> source_metadata.Filesystem + 12, // 26: source_metadata.MetaData.git:type_name -> source_metadata.Git + 23, // 27: source_metadata.MetaData.test:type_name -> source_metadata.Test + 5, // 28: source_metadata.MetaData.buildkite:type_name -> source_metadata.Buildkite + 22, // 29: source_metadata.MetaData.gerrit:type_name -> source_metadata.Gerrit + 24, // 30: source_metadata.MetaData.jenkins:type_name -> source_metadata.Jenkins + 25, // 31: source_metadata.MetaData.teams:type_name -> source_metadata.Teams + 26, // 32: source_metadata.MetaData.artifactory:type_name -> source_metadata.Artifactory + 27, // 33: source_metadata.MetaData.syslog:type_name -> source_metadata.Syslog + 28, // 34: source_metadata.MetaData.forager:type_name -> source_metadata.Forager + 29, // 35: source_metadata.MetaData.sharepoint:type_name -> source_metadata.SharePoint + 30, // 36: source_metadata.MetaData.googleDrive:type_name -> source_metadata.GoogleDrive + 31, // 37: source_metadata.MetaData.azureRepos:type_name -> source_metadata.AzureRepos + 7, // 38: source_metadata.MetaData.travisCI:type_name -> source_metadata.TravisCI + 32, // 39: source_metadata.MetaData.postman:type_name -> source_metadata.Postman + 34, // 40: source_metadata.MetaData.webhook:type_name -> source_metadata.Webhook + 35, // 41: source_metadata.MetaData.elasticsearch:type_name -> source_metadata.Elasticsearch + 16, // 42: source_metadata.MetaData.huggingface:type_name -> source_metadata.Huggingface + 36, // 43: source_metadata.MetaData.sentry:type_name -> source_metadata.Sentry + 37, // 44: source_metadata.MetaData.stdin:type_name -> source_metadata.Stdin + 38, // 45: source_metadata.MetaData.slackContinuous:type_name -> source_metadata.SlackContinuous + 39, // 46: source_metadata.MetaData.jsonEnumerator:type_name -> source_metadata.JSONEnumerator + 2, // 47: source_metadata.MetaData.awsSecretsManager:type_name -> source_metadata.AWSSecretsManager + 48, // [48:48] is the sub-list for method output_type + 48, // [48:48] is the sub-list for method input_type + 48, // [48:48] is the sub-list for extension type_name + 48, // [48:48] is the sub-list for extension extendee + 0, // [0:48] is the sub-list for field type_name } func init() { file_source_metadata_proto_init() } @@ -4881,7 +4979,7 @@ func file_source_metadata_proto_init() { } if !protoimpl.UnsafeEnabled { file_source_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Azure); i { + switch v := v.(*AWSSecretsManager); i { case 0: return &v.state case 1: @@ -4893,7 +4991,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Bitbucket); i { + switch v := v.(*Azure); i { case 0: return &v.state case 1: @@ -4905,7 +5003,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Buildkite); i { + switch v := v.(*Bitbucket); i { case 0: return &v.state case 1: @@ -4917,7 +5015,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CircleCI); i { + switch v := v.(*Buildkite); i { case 0: return &v.state case 1: @@ -4929,7 +5027,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TravisCI); i { + switch v := v.(*CircleCI); i { case 0: return &v.state case 1: @@ -4941,7 +5039,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Confluence); i { + switch v := v.(*TravisCI); i { case 0: return &v.state case 1: @@ -4953,7 +5051,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Docker); i { + switch v := v.(*Confluence); i { case 0: return &v.state case 1: @@ -4965,7 +5063,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ECR); i { + switch v := v.(*Docker); i { case 0: return &v.state case 1: @@ -4977,7 +5075,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Filesystem); i { + switch v := v.(*ECR); i { case 0: return &v.state case 1: @@ -4989,7 +5087,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Git); i { + switch v := v.(*Filesystem); i { case 0: return &v.state case 1: @@ -5001,7 +5099,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Github); i { + switch v := v.(*Git); i { case 0: return &v.state case 1: @@ -5013,7 +5111,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Gitlab); i { + switch v := v.(*Github); i { case 0: return &v.state case 1: @@ -5025,7 +5123,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GCS); i { + switch v := v.(*Gitlab); i { case 0: return &v.state case 1: @@ -5037,7 +5135,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Huggingface); i { + switch v := v.(*GCS); i { case 0: return &v.state case 1: @@ -5049,7 +5147,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Jira); i { + switch v := v.(*Huggingface); i { case 0: return &v.state case 1: @@ -5061,7 +5159,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NPM); i { + switch v := v.(*Jira); i { case 0: return &v.state case 1: @@ -5073,7 +5171,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PyPi); i { + switch v := v.(*NPM); i { case 0: return &v.state case 1: @@ -5085,7 +5183,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*S3); i { + switch v := v.(*PyPi); i { case 0: return &v.state case 1: @@ -5097,7 +5195,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Slack); i { + switch v := v.(*S3); i { case 0: return &v.state case 1: @@ -5109,7 +5207,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Gerrit); i { + switch v := v.(*Slack); i { case 0: return &v.state case 1: @@ -5121,7 +5219,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Test); i { + switch v := v.(*Gerrit); i { case 0: return &v.state case 1: @@ -5133,7 +5231,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Jenkins); i { + switch v := v.(*Test); i { case 0: return &v.state case 1: @@ -5145,7 +5243,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Teams); i { + switch v := v.(*Jenkins); i { case 0: return &v.state case 1: @@ -5157,7 +5255,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Artifactory); i { + switch v := v.(*Teams); i { case 0: return &v.state case 1: @@ -5169,7 +5267,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Syslog); i { + switch v := v.(*Artifactory); i { case 0: return &v.state case 1: @@ -5181,7 +5279,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Forager); i { + switch v := v.(*Syslog); i { case 0: return &v.state case 1: @@ -5193,7 +5291,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SharePoint); i { + switch v := v.(*Forager); i { case 0: return &v.state case 1: @@ -5205,7 +5303,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GoogleDrive); i { + switch v := v.(*SharePoint); i { case 0: return &v.state case 1: @@ -5217,7 +5315,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AzureRepos); i { + switch v := v.(*GoogleDrive); i { case 0: return &v.state case 1: @@ -5229,7 +5327,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Postman); i { + switch v := v.(*AzureRepos); i { case 0: return &v.state case 1: @@ -5241,7 +5339,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Vector); i { + switch v := v.(*Postman); i { case 0: return &v.state case 1: @@ -5253,7 +5351,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Webhook); i { + switch v := v.(*Vector); i { case 0: return &v.state case 1: @@ -5265,7 +5363,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Elasticsearch); i { + switch v := v.(*Webhook); i { case 0: return &v.state case 1: @@ -5277,7 +5375,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Sentry); i { + switch v := v.(*Elasticsearch); i { case 0: return &v.state case 1: @@ -5289,7 +5387,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Stdin); i { + switch v := v.(*Sentry); i { case 0: return &v.state case 1: @@ -5301,7 +5399,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SlackContinuous); i { + switch v := v.(*Stdin); i { case 0: return &v.state case 1: @@ -5313,7 +5411,7 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*JSONEnumerator); i { + switch v := v.(*SlackContinuous); i { case 0: return &v.state case 1: @@ -5325,6 +5423,18 @@ func file_source_metadata_proto_init() { } } file_source_metadata_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*JSONEnumerator); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_source_metadata_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetaData); i { case 0: return &v.state @@ -5337,15 +5447,15 @@ func file_source_metadata_proto_init() { } } } - file_source_metadata_proto_msgTypes[25].OneofWrappers = []interface{}{ + file_source_metadata_proto_msgTypes[26].OneofWrappers = []interface{}{ (*Forager_Github)(nil), (*Forager_Npm)(nil), (*Forager_Pypi)(nil), } - file_source_metadata_proto_msgTypes[31].OneofWrappers = []interface{}{ + file_source_metadata_proto_msgTypes[32].OneofWrappers = []interface{}{ (*Webhook_Vector)(nil), } - file_source_metadata_proto_msgTypes[37].OneofWrappers = []interface{}{ + file_source_metadata_proto_msgTypes[38].OneofWrappers = []interface{}{ (*MetaData_Azure)(nil), (*MetaData_Bitbucket)(nil), (*MetaData_Circleci)(nil), @@ -5382,6 +5492,7 @@ func file_source_metadata_proto_init() { (*MetaData_Stdin)(nil), (*MetaData_SlackContinuous)(nil), (*MetaData_JsonEnumerator)(nil), + (*MetaData_AwsSecretsManager)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -5389,7 +5500,7 @@ func file_source_metadata_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_source_metadata_proto_rawDesc, NumEnums: 2, - NumMessages: 38, + NumMessages: 39, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.validate.go b/pkg/pb/source_metadatapb/source_metadata.pb.validate.go index 1cb13f369b59..ef652fb97e37 100644 --- a/pkg/pb/source_metadatapb/source_metadata.pb.validate.go +++ b/pkg/pb/source_metadatapb/source_metadata.pb.validate.go @@ -35,6 +35,121 @@ var ( _ = sort.Sort ) +// Validate checks the field values on AWSSecretsManager with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *AWSSecretsManager) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on AWSSecretsManager with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// AWSSecretsManagerMultiError, or nil if none found. +func (m *AWSSecretsManager) ValidateAll() error { + return m.validate(true) +} + +func (m *AWSSecretsManager) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if !_AWSSecretsManager_SecretArn_Pattern.MatchString(m.GetSecretArn()) { + err := AWSSecretsManagerValidationError{ + field: "SecretArn", + reason: "value does not match regex pattern \"^arn:aws[^:]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+$\"", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return AWSSecretsManagerMultiError(errors) + } + + return nil +} + +// AWSSecretsManagerMultiError is an error wrapping multiple validation errors +// returned by AWSSecretsManager.ValidateAll() if the designated constraints +// aren't met. +type AWSSecretsManagerMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m AWSSecretsManagerMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m AWSSecretsManagerMultiError) AllErrors() []error { return m } + +// AWSSecretsManagerValidationError is the validation error returned by +// AWSSecretsManager.Validate if the designated constraints aren't met. +type AWSSecretsManagerValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e AWSSecretsManagerValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e AWSSecretsManagerValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e AWSSecretsManagerValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e AWSSecretsManagerValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e AWSSecretsManagerValidationError) ErrorName() string { + return "AWSSecretsManagerValidationError" +} + +// Error satisfies the builtin error interface +func (e AWSSecretsManagerValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sAWSSecretsManager.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = AWSSecretsManagerValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = AWSSecretsManagerValidationError{} + +var _AWSSecretsManager_SecretArn_Pattern = regexp.MustCompile("^arn:aws[^:]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+$") + // Validate checks the field values on Azure with the rules defined in the // proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. @@ -3089,6 +3204,10 @@ func (m *SharePoint) validate(all bool) error { // no validation rules for Email + // no validation rules for Version + + // no validation rules for CommentNumber + if len(errors) > 0 { return SharePointMultiError(errors) } @@ -5848,6 +5967,47 @@ func (m *MetaData) validate(all bool) error { } } + case *MetaData_AwsSecretsManager: + if v == nil { + err := MetaDataValidationError{ + field: "Data", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetAwsSecretsManager()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, MetaDataValidationError{ + field: "AwsSecretsManager", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, MetaDataValidationError{ + field: "AwsSecretsManager", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetAwsSecretsManager()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return MetaDataValidationError{ + field: "AwsSecretsManager", + reason: "embedded message failed validation", + cause: err, + } + } + } + default: _ = v // ensures v is used } diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go index 37424d7eee7c..15b156076a60 100644 --- a/pkg/pb/sourcespb/sources.pb.go +++ b/pkg/pb/sourcespb/sources.pb.go @@ -71,6 +71,7 @@ const ( SourceType_SOURCE_TYPE_STDIN SourceType = 40 SourceType_SOURCE_TYPE_SLACK_CONTINUOUS SourceType = 41 SourceType_SOURCE_TYPE_JSON_ENUMERATOR SourceType = 42 + SourceType_SOURCE_TYPE_AWS_SECRETS_MANAGER SourceType = 43 ) // Enum value maps for SourceType. @@ -119,6 +120,7 @@ var ( 40: "SOURCE_TYPE_STDIN", 41: "SOURCE_TYPE_SLACK_CONTINUOUS", 42: "SOURCE_TYPE_JSON_ENUMERATOR", + 43: "SOURCE_TYPE_AWS_SECRETS_MANAGER", } SourceType_value = map[string]int32{ "SOURCE_TYPE_AZURE_STORAGE": 0, @@ -164,6 +166,7 @@ var ( "SOURCE_TYPE_STDIN": 40, "SOURCE_TYPE_SLACK_CONTINUOUS": 41, "SOURCE_TYPE_JSON_ENUMERATOR": 42, + "SOURCE_TYPE_AWS_SECRETS_MANAGER": 43, } ) @@ -693,6 +696,7 @@ type Bitbucket struct { // *Bitbucket_Token // *Bitbucket_Oauth // *Bitbucket_BasicAuth + // *Bitbucket_ApiToken Credential isBitbucket_Credential `protobuf_oneof:"credential"` Repositories []string `protobuf:"bytes,5,rep,name=repositories,proto3" json:"repositories,omitempty"` IgnoreRepos []string `protobuf:"bytes,6,rep,name=ignore_repos,json=ignoreRepos,proto3" json:"ignore_repos,omitempty"` @@ -772,6 +776,13 @@ func (x *Bitbucket) GetBasicAuth() *credentialspb.BasicAuth { return nil } +func (x *Bitbucket) GetApiToken() *credentialspb.BitbucketApiToken { + if x, ok := x.GetCredential().(*Bitbucket_ApiToken); ok { + return x.ApiToken + } + return nil +} + func (x *Bitbucket) GetRepositories() []string { if x != nil { return x.Repositories @@ -851,12 +862,18 @@ type Bitbucket_BasicAuth struct { BasicAuth *credentialspb.BasicAuth `protobuf:"bytes,4,opt,name=basic_auth,json=basicAuth,proto3,oneof"` } +type Bitbucket_ApiToken struct { + ApiToken *credentialspb.BitbucketApiToken `protobuf:"bytes,14,opt,name=api_token,json=apiToken,proto3,oneof"` +} + func (*Bitbucket_Token) isBitbucket_Credential() {} func (*Bitbucket_Oauth) isBitbucket_Credential() {} func (*Bitbucket_BasicAuth) isBitbucket_Credential() {} +func (*Bitbucket_ApiToken) isBitbucket_Credential() {} + type CircleCI struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2107,6 +2124,9 @@ type GitHub struct { NoCleanup bool `protobuf:"varint,23,opt,name=no_cleanup,json=noCleanup,proto3" json:"no_cleanup,omitempty"` IgnoreGists bool `protobuf:"varint,24,opt,name=ignore_gists,json=ignoreGists,proto3" json:"ignore_gists,omitempty"` PrintLegacyJson bool `protobuf:"varint,25,opt,name=print_legacy_json,json=printLegacyJson,proto3" json:"print_legacy_json,omitempty"` + // When using GitHub App credentials, enumerate repositories from every app installation. + ScanAllInstallations bool `protobuf:"varint,26,opt,name=scan_all_installations,json=scanAllInstallations,proto3" json:"scan_all_installations,omitempty"` + ExcludeArchived bool `protobuf:"varint,27,opt,name=exclude_archived,json=excludeArchived,proto3" json:"exclude_archived,omitempty"` } func (x *GitHub) Reset() { @@ -2323,6 +2343,20 @@ func (x *GitHub) GetPrintLegacyJson() bool { return false } +func (x *GitHub) GetScanAllInstallations() bool { + if x != nil { + return x.ScanAllInstallations + } + return false +} + +func (x *GitHub) GetExcludeArchived() bool { + if x != nil { + return x.ExcludeArchived + } + return false +} + type isGitHub_Credential interface { isGitHub_Credential() } @@ -2719,6 +2753,10 @@ type Huggingface struct { SkipAllDatasets bool `protobuf:"varint,18,opt,name=skip_all_datasets,json=skipAllDatasets,proto3" json:"skip_all_datasets,omitempty"` IncludeDiscussions bool `protobuf:"varint,11,opt,name=include_discussions,json=includeDiscussions,proto3" json:"include_discussions,omitempty"` IncludePrs bool `protobuf:"varint,19,opt,name=include_prs,json=includePrs,proto3" json:"include_prs,omitempty"` + Buckets []string `protobuf:"bytes,20,rep,name=buckets,proto3" json:"buckets,omitempty"` + IgnoreBuckets []string `protobuf:"bytes,21,rep,name=ignore_buckets,json=ignoreBuckets,proto3" json:"ignore_buckets,omitempty"` + IncludeBuckets []string `protobuf:"bytes,22,rep,name=include_buckets,json=includeBuckets,proto3" json:"include_buckets,omitempty"` + SkipAllBuckets bool `protobuf:"varint,23,opt,name=skip_all_buckets,json=skipAllBuckets,proto3" json:"skip_all_buckets,omitempty"` } func (x *Huggingface) Reset() { @@ -2893,6 +2931,34 @@ func (x *Huggingface) GetIncludePrs() bool { return false } +func (x *Huggingface) GetBuckets() []string { + if x != nil { + return x.Buckets + } + return nil +} + +func (x *Huggingface) GetIgnoreBuckets() []string { + if x != nil { + return x.IgnoreBuckets + } + return nil +} + +func (x *Huggingface) GetIncludeBuckets() []string { + if x != nil { + return x.IncludeBuckets + } + return nil +} + +func (x *Huggingface) GetSkipAllBuckets() bool { + if x != nil { + return x.SkipAllBuckets + } + return false +} + type isHuggingface_Credential interface { isHuggingface_Credential() } @@ -3212,6 +3278,25 @@ type S3 struct { IgnoreBuckets []string `protobuf:"bytes,8,rep,name=ignore_buckets,json=ignoreBuckets,proto3" json:"ignore_buckets,omitempty"` // Deprecated: Marked as deprecated in sources.proto. EnableResumption bool `protobuf:"varint,9,opt,name=enable_resumption,json=enableResumption,proto3" json:"enable_resumption,omitempty"` + // endpoint targets an S3-compatible service other than AWS S3. When set, + // per-bucket region discovery is skipped because it is an AWS-only API. + Endpoint string `protobuf:"bytes,10,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + // region is the region used to sign requests. Defaults to us-east-1. + Region string `protobuf:"bytes,11,opt,name=region,proto3" json:"region,omitempty"` + // include_prefixes limits the scan to objects whose key matches one of these + // key prefixes. Leave empty to consider every object in the bucket. + IncludePrefixes []string `protobuf:"bytes,12,rep,name=include_prefixes,json=includePrefixes,proto3" json:"include_prefixes,omitempty"` + // exclude_prefixes skips objects whose key matches one of these key prefixes. + ExcludePrefixes []string `protobuf:"bytes,13,rep,name=exclude_prefixes,json=excludePrefixes,proto3" json:"exclude_prefixes,omitempty"` + // include_extensions limits the scan to objects whose key ends in one of these + // file extensions, written without a leading dot (e.g. "tf", "yaml"). Leave + // empty to consider every extension. Cannot be combined with + // exclude_extensions. + IncludeExtensions []string `protobuf:"bytes,14,rep,name=include_extensions,json=includeExtensions,proto3" json:"include_extensions,omitempty"` + // exclude_extensions skips objects whose key ends in one of these file + // extensions, written without a leading dot (e.g. "zip", "mp4"). Cannot be + // combined with include_extensions. + ExcludeExtensions []string `protobuf:"bytes,15,rep,name=exclude_extensions,json=excludeExtensions,proto3" json:"exclude_extensions,omitempty"` } func (x *S3) Reset() { @@ -3317,6 +3402,48 @@ func (x *S3) GetEnableResumption() bool { return false } +func (x *S3) GetEndpoint() string { + if x != nil { + return x.Endpoint + } + return "" +} + +func (x *S3) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *S3) GetIncludePrefixes() []string { + if x != nil { + return x.IncludePrefixes + } + return nil +} + +func (x *S3) GetExcludePrefixes() []string { + if x != nil { + return x.ExcludePrefixes + } + return nil +} + +func (x *S3) GetIncludeExtensions() []string { + if x != nil { + return x.IncludeExtensions + } + return nil +} + +func (x *S3) GetExcludeExtensions() []string { + if x != nil { + return x.ExcludeExtensions + } + return nil +} + type isS3_Credential interface { isS3_Credential() } @@ -4159,8 +4286,11 @@ type Sharepoint struct { // Types that are assignable to Credential: // // *Sharepoint_Oauth - Credential isSharepoint_Credential `protobuf_oneof:"credential"` - SiteUrl string `protobuf:"bytes,2,opt,name=site_url,json=siteUrl,proto3" json:"site_url,omitempty"` + Credential isSharepoint_Credential `protobuf_oneof:"credential"` + SiteUrl string `protobuf:"bytes,2,opt,name=site_url,json=siteUrl,proto3" json:"site_url,omitempty"` + IncludeVersions bool `protobuf:"varint,3,opt,name=include_versions,json=includeVersions,proto3" json:"include_versions,omitempty"` + IncludeComments bool `protobuf:"varint,4,opt,name=include_comments,json=includeComments,proto3" json:"include_comments,omitempty"` + IncludeAttachments bool `protobuf:"varint,5,opt,name=include_attachments,json=includeAttachments,proto3" json:"include_attachments,omitempty"` } func (x *Sharepoint) Reset() { @@ -4216,6 +4346,27 @@ func (x *Sharepoint) GetSiteUrl() string { return "" } +func (x *Sharepoint) GetIncludeVersions() bool { + if x != nil { + return x.IncludeVersions + } + return false +} + +func (x *Sharepoint) GetIncludeComments() bool { + if x != nil { + return x.IncludeComments + } + return false +} + +func (x *Sharepoint) GetIncludeAttachments() bool { + if x != nil { + return x.IncludeAttachments + } + return false +} + type isSharepoint_Credential interface { isSharepoint_Credential() } @@ -5168,7 +5319,7 @@ var file_sources_proto_rawDesc = []byte{ 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf5, 0x04, 0x0a, 0x09, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, + 0x74, 0x69, 0x61, 0x6c, 0x22, 0xb4, 0x05, 0x0a, 0x09, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, @@ -5179,314 +5330,324 @@ var file_sources_proto_rawDesc = []byte{ 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, - 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, - 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, - 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, - 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, - 0x76, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, - 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x12, 0x4f, 0x0a, 0x11, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x42, 0x69, 0x74, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x40, 0x0a, 0x1c, 0x6f, 0x61, 0x75, 0x74, - 0x68, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, - 0x6f, 0x61, 0x75, 0x74, 0x68, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x6f, 0x61, - 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, - 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x18, 0x0c, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x12, - 0x3d, 0x0a, 0x1b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, - 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x0d, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x57, 0x72, 0x69, 0x74, 0x65, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x56, 0x0a, 0x08, - 0x43, 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, - 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, - 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x22, 0x56, 0x0a, 0x08, 0x54, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, + 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x3d, 0x0a, 0x09, 0x61, 0x70, 0x69, 0x5f, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x41, 0x70, 0x69, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x48, 0x00, 0x52, 0x08, 0x61, 0x70, 0x69, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, + 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, 0x0d, + 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, + 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, + 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x12, 0x4f, 0x0a, 0x11, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, + 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x22, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x42, 0x69, 0x74, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x40, 0x0a, 0x1c, 0x6f, 0x61, 0x75, 0x74, 0x68, + 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x6f, + 0x61, 0x75, 0x74, 0x68, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x6f, 0x61, 0x75, + 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6f, + 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0b, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x73, 0x12, 0x3d, + 0x0a, 0x1b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x5f, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x0d, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x18, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x73, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x57, 0x72, 0x69, 0x74, 0x65, 0x42, 0x0c, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x56, 0x0a, 0x08, 0x43, + 0x69, 0x72, 0x63, 0x6c, 0x65, 0x43, 0x49, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, + 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, + 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x22, 0x56, 0x0a, 0x08, 0x54, 0x72, 0x61, 0x76, 0x69, 0x73, 0x43, 0x49, 0x12, + 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xd2, 0x04, 0x0a, 0x0a, + 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, + 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, + 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, + 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, + 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, + 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, + 0x75, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x48, 0x0a, 0x0c, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x73, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x25, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x66, + 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x53, 0x70, 0x61, + 0x63, 0x65, 0x73, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x52, 0x0b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, + 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, + 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x74, 0x6c, + 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, + 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x54, 0x6c, 0x73, 0x12, 0x16, + 0x0a, 0x06, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, + 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, + 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, + 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x36, 0x0a, 0x11, 0x47, 0x65, + 0x74, 0x41, 0x6c, 0x6c, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, + 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x47, 0x4c, 0x4f, 0x42, + 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x45, 0x52, 0x53, 0x4f, 0x4e, 0x41, 0x4c, + 0x10, 0x02, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x22, 0xeb, 0x02, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x48, 0x0a, 0x0f, 0x75, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, + 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, + 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x23, + 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x29, 0x0a, 0x0f, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x6b, 0x65, + 0x79, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0e, + 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x12, 0x16, + 0x0a, 0x06, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x65, + 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x79, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x6c, + 0x0a, 0x03, 0x45, 0x43, 0x52, 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1e, + 0x0a, 0x0a, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, + 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf1, 0x01, 0x0a, + 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x20, 0x0a, 0x0b, 0x64, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, + 0x74, 0x68, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, + 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, + 0x68, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x65, + 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x12, + 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, + 0x72, 0x69, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x79, 0x6d, 0x6c, + 0x69, 0x6e, 0x6b, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0f, 0x6d, 0x61, 0x78, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x44, 0x65, 0x70, 0x74, 0x68, + 0x22, 0xab, 0x04, 0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x32, 0x0a, 0x14, 0x6a, 0x73, 0x6f, 0x6e, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x12, 0x6a, 0x73, 0x6f, 0x6e, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x07, + 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, + 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, + 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x12, 0x31, 0x0a, 0x03, 0x61, 0x64, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, + 0x03, 0x61, 0x64, 0x63, 0x12, 0x32, 0x0a, 0x14, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x12, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, + 0x68, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, + 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x27, 0x0a, + 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x42, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x27, 0x0a, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, + 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, + 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xe4, + 0x05, 0x0a, 0x03, 0x47, 0x69, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, + 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, + 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, + 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x08, 0x73, 0x73, 0x68, + 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x53, 0x48, 0x41, 0x75, 0x74, + 0x68, 0x48, 0x00, 0x52, 0x07, 0x73, 0x73, 0x68, 0x41, 0x75, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x22, + 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, + 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x61, + 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x62, 0x61, 0x72, 0x65, 0x12, 0x2c, + 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x2c, 0x0a, 0x12, + 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x5f, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x67, 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x6c, 0x6f, 0x62, 0x73, 0x12, + 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x72, 0x69, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x23, + 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, + 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, + 0x69, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, + 0x69, 0x76, 0x65, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, + 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x6f, 0x6e, + 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, + 0x6f, 0x6e, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x5f, 0x63, 0x6c, + 0x65, 0x61, 0x6e, 0x75, 0x70, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x43, + 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x5f, + 0x6c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x12, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0f, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, + 0x6f, 0x6e, 0x12, 0x33, 0x0a, 0x16, 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x5f, 0x67, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x13, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x13, 0x74, 0x72, 0x75, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x47, 0x69, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf2, 0x04, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x4c, 0x61, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xd2, 0x04, 0x0a, - 0x0a, 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x08, 0x65, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, - 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, - 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, 0x0a, 0x0a, 0x62, - 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, + 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, + 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x37, 0x0a, 0x0a, 0x62, + 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, - 0x41, 0x75, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x48, 0x0a, 0x0c, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x43, 0x6f, 0x6e, - 0x66, 0x6c, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x53, 0x70, - 0x61, 0x63, 0x65, 0x73, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x52, 0x0b, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, - 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x74, - 0x6c, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, - 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x54, 0x6c, 0x73, 0x12, - 0x16, 0x0a, 0x06, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x06, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, - 0x65, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, - 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x13, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x21, 0x0a, - 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x0a, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, - 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x36, 0x0a, 0x11, 0x47, - 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x53, 0x63, 0x6f, 0x70, 0x65, - 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x47, 0x4c, 0x4f, - 0x42, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x45, 0x52, 0x53, 0x4f, 0x4e, 0x41, - 0x4c, 0x10, 0x02, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x22, 0xeb, 0x02, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x48, 0x0a, 0x0f, + 0x41, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, + 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, + 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, + 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, + 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, + 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x4c, 0x0a, 0x23, 0x65, 0x78, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x5f, 0x73, + 0x68, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x74, 0x6f, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, + 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x49, 0x6e, + 0x74, 0x6f, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x72, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x69, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x75, 0x74, 0x68, + 0x49, 0x6e, 0x55, 0x72, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, + 0x64, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, + 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x50, 0x61, 0x74, + 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x5f, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x18, + 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, + 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x67, 0x61, 0x63, 0x79, + 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x70, 0x72, 0x69, + 0x6e, 0x74, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xfb, 0x08, 0x0a, 0x06, 0x47, + 0x69, 0x74, 0x48, 0x75, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, + 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x5f, 0x61, 0x70, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x47, 0x69, + 0x74, 0x48, 0x75, 0x62, 0x41, 0x70, 0x70, 0x48, 0x00, 0x52, 0x09, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x41, 0x70, 0x70, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, - 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, + 0x61, 0x75, 0x74, 0x68, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, - 0x23, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x29, 0x0a, 0x0f, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x6b, - 0x65, 0x79, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, - 0x0e, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x12, - 0x16, 0x0a, 0x06, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x06, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, - 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0x6c, 0x0a, 0x03, 0x45, 0x43, 0x52, 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, - 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x69, 0x65, 0x73, 0x42, - 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf1, 0x01, - 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x20, 0x0a, 0x0b, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, - 0x61, 0x74, 0x68, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, - 0x70, 0x61, 0x74, 0x68, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, - 0x6c, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x73, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, - 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, 0x65, - 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, - 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, - 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x79, 0x6d, - 0x6c, 0x69, 0x6e, 0x6b, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0f, 0x6d, 0x61, 0x78, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x44, 0x65, 0x70, 0x74, - 0x68, 0x22, 0xab, 0x04, 0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x32, 0x0a, 0x14, 0x6a, 0x73, 0x6f, - 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x12, 0x6a, 0x73, 0x6f, 0x6e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x0a, - 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, - 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, - 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x64, 0x12, 0x31, 0x0a, 0x03, 0x61, 0x64, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1d, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, - 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, - 0x52, 0x03, 0x61, 0x64, 0x63, 0x12, 0x32, 0x0a, 0x14, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x0b, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x12, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, - 0x74, 0x68, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, - 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x27, - 0x0a, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, - 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, - 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0xe4, 0x05, 0x0a, 0x03, 0x47, 0x69, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, - 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, - 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, - 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, - 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x31, 0x0a, 0x08, 0x73, 0x73, - 0x68, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x53, 0x48, 0x41, 0x75, - 0x74, 0x68, 0x48, 0x00, 0x52, 0x07, 0x73, 0x73, 0x68, 0x41, 0x75, 0x74, 0x68, 0x12, 0x20, 0x0a, - 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, - 0x69, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, - 0x61, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x62, 0x61, 0x72, 0x65, 0x12, - 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, - 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x69, 0x6e, 0x63, - 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x2c, 0x0a, - 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x5f, 0x66, - 0x69, 0x6c, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, - 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x67, 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0c, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x6c, 0x6f, 0x62, 0x73, - 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x72, 0x69, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, - 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, - 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, - 0x72, 0x69, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, - 0x68, 0x69, 0x76, 0x65, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, - 0x70, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x6f, - 0x6e, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, - 0x6c, 0x6f, 0x6e, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x5f, 0x63, - 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, - 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x72, 0x69, 0x6e, 0x74, - 0x5f, 0x6c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x12, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0f, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, - 0x73, 0x6f, 0x6e, 0x12, 0x33, 0x0a, 0x16, 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x5f, 0x67, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x13, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x13, 0x74, 0x72, 0x75, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x47, - 0x69, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf2, 0x04, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x4c, 0x61, - 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, - 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, - 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, - 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x37, 0x0a, 0x0a, - 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, - 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, - 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, 0x0d, - 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, - 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, - 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, - 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, - 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x4c, 0x0a, 0x23, 0x65, - 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x5f, - 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x74, 0x6f, 0x5f, 0x67, 0x72, 0x6f, 0x75, - 0x70, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x49, - 0x6e, 0x74, 0x6f, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x72, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x69, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x75, 0x74, - 0x68, 0x49, 0x6e, 0x55, 0x72, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, - 0x69, 0x64, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x49, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x5f, 0x70, 0x61, 0x74, - 0x68, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x50, 0x61, - 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x5f, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, - 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, - 0x70, 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x67, 0x61, 0x63, - 0x79, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x70, 0x72, - 0x69, 0x6e, 0x74, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x42, 0x0c, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x9a, 0x08, 0x0a, 0x06, - 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, - 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x5f, 0x61, 0x70, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x47, - 0x69, 0x74, 0x48, 0x75, 0x62, 0x41, 0x70, 0x70, 0x48, 0x00, 0x52, 0x09, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x41, 0x70, 0x70, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x48, 0x0a, - 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, - 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, - 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, - 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, - 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, - 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, - 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x63, - 0x61, 0x6e, 0x55, 0x73, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x73, - 0x63, 0x61, 0x6e, 0x55, 0x73, 0x65, 0x72, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x12, 0x0a, 0x04, - 0x68, 0x65, 0x61, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x62, 0x61, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, - 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x41, 0x0a, 0x1d, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x1a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x75, 0x6c, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x69, 0x73, 0x73, 0x75, 0x65, - 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, - 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x5f, 0x67, 0x69, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x10, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, - 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, - 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x23, - 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x18, - 0x12, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, 0x63, 0x68, 0x69, - 0x76, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x77, - 0x69, 0x6b, 0x69, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x57, 0x69, 0x6b, 0x69, 0x73, 0x12, 0x36, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x64, - 0x61, 0x79, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x63, 0x6f, 0x6d, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x44, 0x61, 0x79, 0x73, - 0x12, 0x2b, 0x0a, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, - 0x69, 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x41, 0x75, 0x74, 0x68, 0x49, 0x6e, 0x55, 0x72, 0x6c, 0x12, 0x1d, 0x0a, - 0x0a, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x16, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, - 0x6e, 0x6f, 0x5f, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x6e, 0x6f, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x67, 0x69, 0x73, 0x74, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x47, 0x69, 0x73, 0x74, 0x73, 0x12, 0x2a, - 0x0a, 0x11, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x5f, 0x6a, - 0x73, 0x6f, 0x6e, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x70, 0x72, 0x69, 0x6e, 0x74, - 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, + 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, + 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x63, 0x61, + 0x6e, 0x55, 0x73, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x73, 0x63, + 0x61, 0x6e, 0x55, 0x73, 0x65, 0x72, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x68, + 0x65, 0x61, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x61, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x62, + 0x61, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, + 0x70, 0x6f, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, + 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x41, 0x0a, 0x1d, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x1a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x34, + 0x0a, 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x69, 0x73, 0x73, 0x75, 0x65, 0x5f, + 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, + 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, + 0x67, 0x69, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x10, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, + 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, + 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x23, 0x0a, + 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x18, 0x12, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x77, 0x69, + 0x6b, 0x69, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x57, 0x69, 0x6b, 0x69, 0x73, 0x12, 0x36, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x64, 0x61, + 0x79, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x44, 0x61, 0x79, 0x73, 0x12, + 0x2b, 0x0a, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x69, + 0x6e, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x15, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x41, 0x75, 0x74, 0x68, 0x49, 0x6e, 0x55, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, + 0x6f, 0x5f, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x6e, 0x6f, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, + 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x67, 0x69, 0x73, 0x74, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x47, 0x69, 0x73, 0x74, 0x73, 0x12, 0x2a, 0x0a, + 0x11, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x5f, 0x6a, 0x73, + 0x6f, 0x6e, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x4c, + 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x16, 0x73, 0x63, 0x61, + 0x6e, 0x5f, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x73, 0x63, 0x61, 0x6e, 0x41, + 0x6c, 0x6c, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x29, 0x0a, 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, + 0x76, 0x65, 0x64, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xe4, 0x01, 0x0a, 0x12, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, @@ -5537,7 +5698,7 @@ var file_sources_proto_rawDesc = []byte{ 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x44, 0x57, 0x44, 0x48, 0x00, 0x52, 0x03, 0x64, 0x77, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xe5, 0x05, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf9, 0x06, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x63, 0x65, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, @@ -5583,410 +5744,445 @@ var file_sources_proto_rawDesc = []byte{ 0x63, 0x6c, 0x75, 0x64, 0x65, 0x44, 0x69, 0x73, 0x63, 0x75, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, - 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0xcc, 0x03, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, - 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, - 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, - 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, - 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, - 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, - 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, - 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x16, - 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x69, - 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x76, 0x65, 0x72, - 0x69, 0x66, 0x79, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x69, - 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, - 0x79, 0x54, 0x6c, 0x73, 0x12, 0x4a, 0x0a, 0x11, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x1d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x10, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, - 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, - 0x0a, 0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, - 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, + 0x73, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x14, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x15, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x42, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x16, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x63, + 0x6c, 0x75, 0x64, 0x65, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x73, + 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x6c, 0x6c, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, + 0x17, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x6c, 0x6c, 0x42, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x22, 0xcc, 0x03, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, + 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, + 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, - 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xde, 0x03, 0x0a, 0x02, 0x53, 0x33, - 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, - 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, - 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, + 0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, + 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, + 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x15, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, + 0x65, 0x72, 0x69, 0x66, 0x79, 0x54, 0x6c, 0x73, 0x12, 0x4a, 0x0a, 0x11, 0x69, 0x6e, 0x73, 0x74, + 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x4a, 0x69, + 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x22, 0x73, 0x0a, 0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, + 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, + 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, + 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, + 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, + 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, + 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, + 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xd0, 0x05, + 0x0a, 0x02, 0x53, 0x33, 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, + 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, + 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, + 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, + 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x48, 0x00, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, + 0x78, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, + 0x2f, 0x0a, 0x11, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x10, + 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x29, + 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x65, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, 0x63, + 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x0d, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x65, 0x66, + 0x69, 0x78, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x11, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x22, 0xc4, 0x01, 0x0a, 0x05, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, + 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, + 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x22, 0xa5, 0x02, 0x0a, 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, + 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, + 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, + 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, + 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, + 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, + 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, + 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, + 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, + 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa8, 0x02, 0x0a, 0x07, 0x4a, + 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, + 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, + 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, + 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, + 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, + 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, + 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, + 0x0a, 0x18, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, + 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x15, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, + 0x72, 0x69, 0x66, 0x79, 0x54, 0x6c, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xbd, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, + 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, + 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, + 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1f, + 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x07, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, + 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, + 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x95, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, + 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, + 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xca, 0x01, + 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x72, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, + 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, - 0x10, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, - 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x0d, 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x72, - 0x6f, 0x6c, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x69, 0x67, - 0x6e, 0x6f, 0x72, 0x65, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x11, 0x65, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x10, 0x65, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc4, 0x01, 0x0a, 0x05, 0x53, - 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, - 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, - 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, - 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, - 0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x02, 0x0a, - 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, - 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, - 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, - 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, - 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, - 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, - 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, - 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, - 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, - 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, - 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa8, 0x02, 0x0a, 0x07, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, - 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, - 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, - 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, - 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x48, + 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x1b, 0x0a, + 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, + 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, + 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, + 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xe9, 0x01, + 0x0a, 0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, + 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, + 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, + 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, + 0x65, 0x55, 0x72, 0x6c, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, + 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x69, 0x6e, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, + 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, + 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf6, 0x03, 0x0a, 0x0a, 0x41, 0x7a, + 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, + 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, + 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, + 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, + 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, + 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, + 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, + 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, + 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x21, + 0x0a, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, + 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x70, + 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, + 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, + 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, 0x63, 0x68, + 0x69, 0x76, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x22, 0xd5, 0x04, 0x0a, 0x07, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, - 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x73, 0x65, - 0x63, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, - 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x69, 0x6e, 0x73, 0x65, - 0x63, 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x54, 0x6c, - 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0xbd, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, - 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, - 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, - 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, - 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, - 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x48, 0x00, - 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, - 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, - 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, - 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, - 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, - 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, - 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61, - 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x61, - 0x6d, 0x49, 0x64, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, - 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0x95, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, - 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, - 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, - 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xca, 0x01, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x72, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, - 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, - 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, - 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, - 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, - 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, - 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, - 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, 0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, - 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf6, 0x03, 0x0a, 0x0a, - 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, - 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, - 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, - 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, - 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, - 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, - 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, - 0x70, 0x6f, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, - 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, - 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, - 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, - 0x65, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x41, 0x72, - 0x63, 0x68, 0x69, 0x76, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x22, 0xd5, 0x04, 0x0a, 0x07, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, - 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, - 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, - 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6e, 0x76, 0x69, - 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x65, 0x78, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x43, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x31, 0x0a, 0x14, 0x65, 0x78, 0x63, - 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x31, 0x0a, - 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, - 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x69, 0x6e, 0x63, - 0x6c, 0x75, 0x64, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, - 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, - 0x77, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x64, 0x65, 0x74, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x4b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x27, 0x0a, - 0x0f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, - 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, - 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x65, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, 0x73, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xac, 0x01, 0x0a, - 0x07, 0x57, 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x12, 0x2e, 0x0a, 0x0e, 0x6c, 0x69, 0x73, 0x74, - 0x65, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x42, 0x07, 0xfa, 0x42, 0x04, 0x72, 0x02, 0x68, 0x01, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, - 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x29, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x73, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x48, 0x01, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x42, 0x09, 0x0a, 0x07, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x22, 0x4e, 0x0a, 0x06, 0x56, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, - 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x6f, - 0x63, 0x61, 0x74, 0x6f, 0x72, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, - 0x6e, 0x6b, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x6c, 0x69, 0x6e, 0x6b, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xcd, 0x02, 0x0a, 0x0d, - 0x45, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x14, 0x0a, - 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x6f, - 0x64, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, - 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, - 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x70, 0x61, - 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x64, - 0x65, 0x78, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x69, 0x6e, 0x63, - 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x12, 0x28, 0x0a, 0x10, 0x62, 0x65, 0x73, 0x74, 0x5f, 0x65, 0x66, 0x66, 0x6f, 0x72, 0x74, - 0x5f, 0x73, 0x63, 0x61, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x62, 0x65, 0x73, - 0x74, 0x45, 0x66, 0x66, 0x6f, 0x72, 0x74, 0x53, 0x63, 0x61, 0x6e, 0x22, 0xde, 0x01, 0x0a, 0x06, - 0x53, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x19, 0x0a, 0x07, 0x64, 0x73, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x64, 0x73, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x19, - 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x73, - 0x65, 0x63, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, - 0x79, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x69, 0x6e, 0x73, - 0x65, 0x63, 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x54, - 0x6c, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x07, 0x0a, 0x05, - 0x53, 0x74, 0x64, 0x69, 0x6e, 0x22, 0x4e, 0x0a, 0x0f, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x43, 0x6f, - 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, - 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x2a, 0xc4, 0x09, - 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, - 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, - 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, - 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, - 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, - 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, - 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, - 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, - 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, - 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, - 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, - 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, - 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, - 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, - 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, - 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, - 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, - 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, - 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, - 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, - 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, - 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, - 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, - 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, - 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, - 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, - 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, - 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, - 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, - 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, - 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, - 0x4f, 0x53, 0x10, 0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, - 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, - 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, - 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x45, 0x4c, 0x41, 0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, - 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, - 0x55, 0x47, 0x47, 0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, + 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x1e, 0x0a, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, + 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x43, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x31, 0x0a, 0x14, 0x65, 0x78, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x69, 0x6e, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, + 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x31, 0x0a, 0x14, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2b, + 0x0a, 0x11, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x77, 0x6f, + 0x72, 0x64, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x64, 0x65, 0x74, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x4b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0b, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, + 0x61, 0x74, 0x68, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, + 0x2b, 0x0a, 0x11, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x65, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xac, 0x01, 0x0a, 0x07, 0x57, + 0x65, 0x62, 0x68, 0x6f, 0x6f, 0x6b, 0x12, 0x2e, 0x0a, 0x0e, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, + 0xfa, 0x42, 0x04, 0x72, 0x02, 0x68, 0x01, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x29, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, + 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x48, 0x01, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x09, + 0x0a, 0x07, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x22, 0x4e, 0x0a, 0x06, 0x56, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x6f, 0x63, 0x61, + 0x74, 0x6f, 0x72, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x69, 0x6e, 0x6b, + 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6c, + 0x69, 0x6e, 0x6b, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xcd, 0x02, 0x0a, 0x0d, 0x45, 0x6c, + 0x61, 0x73, 0x74, 0x69, 0x63, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x6e, + 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, + 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x23, 0x0a, + 0x0d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x74, 0x74, + 0x65, 0x72, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, + 0x28, 0x0a, 0x10, 0x62, 0x65, 0x73, 0x74, 0x5f, 0x65, 0x66, 0x66, 0x6f, 0x72, 0x74, 0x5f, 0x73, + 0x63, 0x61, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x62, 0x65, 0x73, 0x74, 0x45, + 0x66, 0x66, 0x6f, 0x72, 0x74, 0x53, 0x63, 0x61, 0x6e, 0x22, 0xde, 0x01, 0x0a, 0x06, 0x53, 0x65, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x1f, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x12, 0x19, 0x0a, 0x07, 0x64, 0x73, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x64, 0x73, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x19, 0x0a, 0x07, + 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x06, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x12, 0x37, 0x0a, 0x18, 0x69, 0x6e, 0x73, 0x65, 0x63, + 0x75, 0x72, 0x65, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, + 0x74, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x69, 0x6e, 0x73, 0x65, 0x63, + 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x54, 0x6c, 0x73, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x07, 0x0a, 0x05, 0x53, 0x74, + 0x64, 0x69, 0x6e, 0x22, 0x4e, 0x0a, 0x0f, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x74, + 0x69, 0x6e, 0x75, 0x6f, 0x75, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, + 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x2a, 0xe9, 0x09, 0x0a, 0x0a, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, + 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, + 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, + 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, + 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, + 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, - 0x55, 0x42, 0x5f, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, - 0x25, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, - 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, - 0x28, 0x12, 0x20, 0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, - 0x53, 0x10, 0x29, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, - 0x4f, 0x52, 0x10, 0x2a, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, - 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87, 0x01, - 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, - 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20, 0x0a, - 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, - 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, - 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, - 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, - 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, + 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, + 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, + 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, + 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, + 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, + 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, + 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, + 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, + 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, + 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, + 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, + 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, + 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, + 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, + 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, + 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, + 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, + 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, + 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, + 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, + 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, + 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, + 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, + 0x10, 0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, + 0x4d, 0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, + 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, + 0x41, 0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, + 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, + 0x47, 0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, + 0x5f, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, + 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, + 0x45, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, + 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, + 0x20, 0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, + 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, + 0x29, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, + 0x10, 0x2a, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x41, 0x57, 0x53, 0x5f, 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x53, 0x5f, 0x4d, 0x41, + 0x4e, 0x41, 0x47, 0x45, 0x52, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, + 0x43, 0x54, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, + 0x0f, 0x0a, 0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, + 0x2a, 0x87, 0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, + 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, + 0x12, 0x20, 0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, + 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, + 0x10, 0x01, 0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, + 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, + 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, + 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, + 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -6054,16 +6250,17 @@ var file_sources_proto_goTypes = []interface{}{ (*credentialspb.BasicAuth)(nil), // 47: credentials.BasicAuth (*credentialspb.Unauthenticated)(nil), // 48: credentials.Unauthenticated (*credentialspb.Oauth2)(nil), // 49: credentials.Oauth2 - (*credentialspb.KeySecret)(nil), // 50: credentials.KeySecret - (*credentialspb.CloudEnvironment)(nil), // 51: credentials.CloudEnvironment - (*credentialspb.SSHAuth)(nil), // 52: credentials.SSHAuth - (*credentialspb.GitHubApp)(nil), // 53: credentials.GitHubApp - (*credentialspb.GoogleDriveDWD)(nil), // 54: credentials.GoogleDriveDWD - (*credentialspb.AWSSessionTokenSecret)(nil), // 55: credentials.AWSSessionTokenSecret - (*credentialspb.SlackTokens)(nil), // 56: credentials.SlackTokens - (*credentialspb.Header)(nil), // 57: credentials.Header - (*credentialspb.ClientCredentials)(nil), // 58: credentials.ClientCredentials - (*timestamppb.Timestamp)(nil), // 59: google.protobuf.Timestamp + (*credentialspb.BitbucketApiToken)(nil), // 50: credentials.BitbucketApiToken + (*credentialspb.KeySecret)(nil), // 51: credentials.KeySecret + (*credentialspb.CloudEnvironment)(nil), // 52: credentials.CloudEnvironment + (*credentialspb.SSHAuth)(nil), // 53: credentials.SSHAuth + (*credentialspb.GitHubApp)(nil), // 54: credentials.GitHubApp + (*credentialspb.GoogleDriveDWD)(nil), // 55: credentials.GoogleDriveDWD + (*credentialspb.AWSSessionTokenSecret)(nil), // 56: credentials.AWSSessionTokenSecret + (*credentialspb.SlackTokens)(nil), // 57: credentials.SlackTokens + (*credentialspb.Header)(nil), // 58: credentials.Header + (*credentialspb.ClientCredentials)(nil), // 59: credentials.ClientCredentials + (*timestamppb.Timestamp)(nil), // 60: google.protobuf.Timestamp } var file_sources_proto_depIdxs = []int32{ 45, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration @@ -6074,61 +6271,62 @@ var file_sources_proto_depIdxs = []int32{ 48, // 5: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated 49, // 6: sources.Bitbucket.oauth:type_name -> credentials.Oauth2 47, // 7: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth - 1, // 8: sources.Bitbucket.installation_type:type_name -> sources.BitbucketInstallationType - 48, // 9: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 10: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth - 3, // 11: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope - 48, // 12: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 13: sources.Docker.basic_auth:type_name -> credentials.BasicAuth - 50, // 14: sources.ECR.access_key:type_name -> credentials.KeySecret - 48, // 15: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated - 51, // 16: sources.GCS.adc:type_name -> credentials.CloudEnvironment - 49, // 17: sources.GCS.oauth:type_name -> credentials.Oauth2 - 47, // 18: sources.Git.basic_auth:type_name -> credentials.BasicAuth - 48, // 19: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated - 52, // 20: sources.Git.ssh_auth:type_name -> credentials.SSHAuth - 49, // 21: sources.GitLab.oauth:type_name -> credentials.Oauth2 - 47, // 22: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth - 53, // 23: sources.GitHub.github_app:type_name -> credentials.GitHubApp - 48, // 24: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 25: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth - 53, // 26: sources.GitHubRealtime.github_app:type_name -> credentials.GitHubApp - 48, // 27: sources.GitHubRealtime.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 28: sources.GitHubRealtime.basic_auth:type_name -> credentials.BasicAuth - 49, // 29: sources.GoogleDrive.oauth:type_name -> credentials.Oauth2 - 54, // 30: sources.GoogleDrive.dwd:type_name -> credentials.GoogleDriveDWD - 48, // 31: sources.Huggingface.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 32: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth - 48, // 33: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated - 49, // 34: sources.JIRA.oauth:type_name -> credentials.Oauth2 - 2, // 35: sources.JIRA.installation_type:type_name -> sources.JiraInstallationType - 48, // 36: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated - 48, // 37: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated - 50, // 38: sources.S3.access_key:type_name -> credentials.KeySecret - 48, // 39: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated - 51, // 40: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment - 55, // 41: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret - 56, // 42: sources.Slack.tokens:type_name -> credentials.SlackTokens - 47, // 43: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth - 48, // 44: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated - 47, // 45: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth - 57, // 46: sources.Jenkins.header:type_name -> credentials.Header - 48, // 47: sources.Jenkins.unauthenticated:type_name -> credentials.Unauthenticated - 58, // 48: sources.Teams.authenticated:type_name -> credentials.ClientCredentials - 49, // 49: sources.Teams.oauth:type_name -> credentials.Oauth2 - 48, // 50: sources.Forager.unauthenticated:type_name -> credentials.Unauthenticated - 59, // 51: sources.Forager.since:type_name -> google.protobuf.Timestamp - 56, // 52: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens - 49, // 53: sources.Sharepoint.oauth:type_name -> credentials.Oauth2 - 49, // 54: sources.AzureRepos.oauth:type_name -> credentials.Oauth2 - 48, // 55: sources.Postman.unauthenticated:type_name -> credentials.Unauthenticated - 57, // 56: sources.Webhook.header:type_name -> credentials.Header - 39, // 57: sources.Webhook.vector:type_name -> sources.Vector - 58, // [58:58] is the sub-list for method output_type - 58, // [58:58] is the sub-list for method input_type - 58, // [58:58] is the sub-list for extension type_name - 58, // [58:58] is the sub-list for extension extendee - 0, // [0:58] is the sub-list for field type_name + 50, // 8: sources.Bitbucket.api_token:type_name -> credentials.BitbucketApiToken + 1, // 9: sources.Bitbucket.installation_type:type_name -> sources.BitbucketInstallationType + 48, // 10: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated + 47, // 11: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth + 3, // 12: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope + 48, // 13: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated + 47, // 14: sources.Docker.basic_auth:type_name -> credentials.BasicAuth + 51, // 15: sources.ECR.access_key:type_name -> credentials.KeySecret + 48, // 16: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated + 52, // 17: sources.GCS.adc:type_name -> credentials.CloudEnvironment + 49, // 18: sources.GCS.oauth:type_name -> credentials.Oauth2 + 47, // 19: sources.Git.basic_auth:type_name -> credentials.BasicAuth + 48, // 20: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated + 53, // 21: sources.Git.ssh_auth:type_name -> credentials.SSHAuth + 49, // 22: sources.GitLab.oauth:type_name -> credentials.Oauth2 + 47, // 23: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth + 54, // 24: sources.GitHub.github_app:type_name -> credentials.GitHubApp + 48, // 25: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated + 47, // 26: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth + 54, // 27: sources.GitHubRealtime.github_app:type_name -> credentials.GitHubApp + 48, // 28: sources.GitHubRealtime.unauthenticated:type_name -> credentials.Unauthenticated + 47, // 29: sources.GitHubRealtime.basic_auth:type_name -> credentials.BasicAuth + 49, // 30: sources.GoogleDrive.oauth:type_name -> credentials.Oauth2 + 55, // 31: sources.GoogleDrive.dwd:type_name -> credentials.GoogleDriveDWD + 48, // 32: sources.Huggingface.unauthenticated:type_name -> credentials.Unauthenticated + 47, // 33: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth + 48, // 34: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated + 49, // 35: sources.JIRA.oauth:type_name -> credentials.Oauth2 + 2, // 36: sources.JIRA.installation_type:type_name -> sources.JiraInstallationType + 48, // 37: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated + 48, // 38: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated + 51, // 39: sources.S3.access_key:type_name -> credentials.KeySecret + 48, // 40: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated + 52, // 41: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment + 56, // 42: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret + 57, // 43: sources.Slack.tokens:type_name -> credentials.SlackTokens + 47, // 44: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth + 48, // 45: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated + 47, // 46: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth + 58, // 47: sources.Jenkins.header:type_name -> credentials.Header + 48, // 48: sources.Jenkins.unauthenticated:type_name -> credentials.Unauthenticated + 59, // 49: sources.Teams.authenticated:type_name -> credentials.ClientCredentials + 49, // 50: sources.Teams.oauth:type_name -> credentials.Oauth2 + 48, // 51: sources.Forager.unauthenticated:type_name -> credentials.Unauthenticated + 60, // 52: sources.Forager.since:type_name -> google.protobuf.Timestamp + 57, // 53: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens + 49, // 54: sources.Sharepoint.oauth:type_name -> credentials.Oauth2 + 49, // 55: sources.AzureRepos.oauth:type_name -> credentials.Oauth2 + 48, // 56: sources.Postman.unauthenticated:type_name -> credentials.Unauthenticated + 58, // 57: sources.Webhook.header:type_name -> credentials.Header + 39, // 58: sources.Webhook.vector:type_name -> sources.Vector + 59, // [59:59] is the sub-list for method output_type + 59, // [59:59] is the sub-list for method input_type + 59, // [59:59] is the sub-list for extension type_name + 59, // [59:59] is the sub-list for extension extendee + 0, // [0:59] is the sub-list for field type_name } func init() { file_sources_proto_init() } @@ -6645,6 +6843,7 @@ func file_sources_proto_init() { (*Bitbucket_Token)(nil), (*Bitbucket_Oauth)(nil), (*Bitbucket_BasicAuth)(nil), + (*Bitbucket_ApiToken)(nil), } file_sources_proto_msgTypes[4].OneofWrappers = []interface{}{ (*CircleCI_Token)(nil), diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go index eca7b0d6a469..0edf9d834acb 100644 --- a/pkg/pb/sourcespb/sources.pb.validate.go +++ b/pkg/pb/sourcespb/sources.pb.validate.go @@ -761,6 +761,47 @@ func (m *Bitbucket) validate(all bool) error { } } + case *Bitbucket_ApiToken: + if v == nil { + err := BitbucketValidationError{ + field: "Credential", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetApiToken()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, BitbucketValidationError{ + field: "ApiToken", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, BitbucketValidationError{ + field: "ApiToken", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetApiToken()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return BitbucketValidationError{ + field: "ApiToken", + reason: "embedded message failed validation", + cause: err, + } + } + } + default: _ = v // ensures v is used } @@ -2590,6 +2631,10 @@ func (m *GitHub) validate(all bool) error { // no validation rules for PrintLegacyJson + // no validation rules for ScanAllInstallations + + // no validation rules for ExcludeArchived + switch v := m.Credential.(type) { case *GitHub_GithubApp: if v == nil { @@ -3444,6 +3489,8 @@ func (m *Huggingface) validate(all bool) error { // no validation rules for IncludePrs + // no validation rules for SkipAllBuckets + switch v := m.Credential.(type) { case *Huggingface_Token: if v == nil { @@ -4154,6 +4201,20 @@ func (m *S3) validate(all bool) error { // no validation rules for EnableResumption + if _, err := url.Parse(m.GetEndpoint()); err != nil { + err = S3ValidationError{ + field: "Endpoint", + reason: "value must be a valid URI", + cause: err, + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Region + switch v := m.Credential.(type) { case *S3_AccessKey: if v == nil { @@ -5887,6 +5948,12 @@ func (m *Sharepoint) validate(all bool) error { // no validation rules for SiteUrl + // no validation rules for IncludeVersions + + // no validation rules for IncludeComments + + // no validation rules for IncludeAttachments + switch v := m.Credential.(type) { case *Sharepoint_Oauth: if v == nil { diff --git a/pkg/roundtripper/roundtripper.go b/pkg/roundtripper/roundtripper.go index d81b7a89a97b..acaec85b171e 100644 --- a/pkg/roundtripper/roundtripper.go +++ b/pkg/roundtripper/roundtripper.go @@ -73,7 +73,7 @@ func (r *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { ) if r.BasicAuthRoundTripper != nil { - req.SetBasicAuth(r.BasicAuthRoundTripper.username, r.BasicAuthRoundTripper.password) + req.SetBasicAuth(r.username, r.password) } authMethod := determineAuth(req) @@ -94,9 +94,9 @@ func (r *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { ) } - reason, shouldRetry, duration := r.RetryableRoundtripper.shouldRetryRequest(response, err) + reason, shouldRetry, duration := r.shouldRetryRequest(response, err) if shouldRetry { - if retries >= int(r.RetryableRoundtripper.maxRetries) { + if retries >= int(r.maxRetries) { r.logger.V(2).Info("max retries reached", "host", req.Host, "reason", reason, diff --git a/pkg/sanitizer/utf8.go b/pkg/sanitizer/utf8.go index 8ebe39834c93..7fe8b7957ad7 100644 --- a/pkg/sanitizer/utf8.go +++ b/pkg/sanitizer/utf8.go @@ -5,5 +5,5 @@ import ( ) func UTF8(in string) string { - return strings.Replace(strings.ToValidUTF8(in, "❗"), "\x00", "", -1) + return strings.ReplaceAll(strings.ToValidUTF8(in, "❗"), "\x00", "") } diff --git a/pkg/sources/chunker_test.go b/pkg/sources/chunker_test.go index ce14cd269ed2..e04fb4539336 100644 --- a/pkg/sources/chunker_test.go +++ b/pkg/sources/chunker_test.go @@ -159,7 +159,7 @@ type panicReader struct{} var _ io.Reader = (*panicReader)(nil) -func (_ panicReader) Read([]byte) (int, error) { +func (panicReader) Read([]byte) (int, error) { panic("panic for testing") } diff --git a/pkg/sources/circleci/README.md b/pkg/sources/circleci/README.md new file mode 100644 index 000000000000..9f63901f3145 --- /dev/null +++ b/pkg/sources/circleci/README.md @@ -0,0 +1,124 @@ +# CircleCI Source + +## Overview + +The CircleCI source enables TruffleHog to scan CircleCI build data for secrets, credentials, and sensitive data. It scans every project accessible by the provided API token, walking each project's build history, job steps, and build configuration. + +## CircleCI Fundamentals + +### What is CircleCI? + +CircleCI is a continuous integration and delivery (CI/CD) platform that automates building, testing, and deploying software. Pipelines are triggered by VCS events (e.g., a GitHub or Bitbucket push) and run a sequence of jobs and steps defined in a project's `config.yml`. + +### Key CircleCI Terminology + +| Term | Description | +|------|-------------| +| **Project** | A repository connected to CircleCI, identified by VCS type, org/username, and repo name | +| **Build** | A single execution of a project's pipeline, identified by an incrementing build number | +| **Job** | A defined unit of work within a build (e.g., `build`, `test`, `deploy`) | +| **Step** | An individual command or action executed within a job | +| **Action** | The underlying execution unit CircleCI's v1.1 API returns per step, with a link to its output | +| **circle.yml** | The build configuration used for a given build (legacy v1.1 naming for `config.yml`) | +| **API Token** | A personal or project token used to authenticate against the CircleCI API | + +## Features + +- **Full Project Discovery**: Automatically lists every project visible to the provided token +- **Build History Scanning**: Iterates through all builds for each discovered project +- **Job Step Output Scanning**: Fetches and scans the output of every action within every build step +- **Build Configuration Scanning**: Scans the `circle.yml`/`config.yml` used for each build +- **Concurrent Processing**: Projects are scanned in parallel using a bounded worker pool +- **CIRCLE_SHA1 Redaction**: Strips the `CIRCLE_SHA1=` environment line from scanned output to avoid noisy commit-SHA false positives + +## Configuration + +### Authentication + +CircleCI only supports token-based authentication. + +**CLI Usage:** +```bash +trufflehog circleci --token cci_xxxxxxxxxxxxxxxxxxxx +``` + +The token can also be provided via the `CIRCLECI_TOKEN` environment variable: + +```bash +export CIRCLECI_TOKEN=cci_xxxxxxxxxxxxxxxxxxxx +trufflehog circleci +``` + +**YAML Configuration:** + +This is the correct shape for a CircleCI entry in a `--config` file: + +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.CircleCI + token: "cci_xxxxxxxxxxxxxxxxxxxx" + name: circleci-scan + type: SOURCE_TYPE_CIRCLECI + verify: true +``` + +Note: as of this writing, `pkg/config/config.go` does not have a case for `SOURCE_TYPE_CIRCLECI` in its source loader, so a config file entry like the one above will fail to load with `got unexpected source type`. Use the CLI flags or environment variable above until that is added. + +## How Scanning Works + +### Scanning Process + +1. **Project Listing**: Fetches all projects the token can access via `GET /projects` +2. **Build Listing**: For each project, lists all builds via `GET /project/:vcs/:org/:repo` +3. **Job/Step Retrieval**: For each build, fetches step and action details via `GET /project/:vcs/:org/:repo/:build_num` +4. **Action Output Scanning**: Downloads and scans the output of every action's `output_url` +5. **Build Config Scanning**: Scans the build's `circle.yml` content +6. **Chunk Generation**: Emits chunks of data to the detection engine, tagged with VCS type, username, repository, build number, and step name + +### What Gets Scanned + +- **Job Step Output**: stdout/stderr captured for each executed step +- **Build Configuration**: The `circle.yml`/`config.yml` content used for each build + +### What Doesn't Get Scanned + +- Artifacts, checkout keys, and environment variables (not currently implemented; see `TODO` in source) +- Lines containing `CIRCLE_SHA1=` (explicitly stripped before scanning) + +## Usage Examples + +### Scanning All Accessible Projects + +```bash +trufflehog circleci --token cci_xxxxxxxxxxxxxxxxxxxx +``` + +### Scanning Using an Environment Variable Token + +```bash +export CIRCLECI_TOKEN=cci_xxxxxxxxxxxxxxxxxxxx +trufflehog circleci +``` + +## Troubleshooting + +### Common Issues + +**Issue**: `invalid credentials, status 401` when listing projects +**Solution**: Verify the token is correct and has not been revoked. Regenerate a personal API token from CircleCI account settings if needed. + +--- + +**Issue**: `no builds found for project` errors for some projects +**Solution**: This is expected for projects with no build history; scanning continues for other projects. + +--- + +**Issue**: Slow scanning with many projects +**Solution**: Increase concurrency via the `concurrency` setting used when initializing the source; scanning is parallelized per project. + +--- + +**Issue**: Missing artifacts, checkout keys, or environment variables in scan results +**Solution**: These are not yet scanned by this source (tracked as a `TODO`); only build step output and build configuration are covered today. diff --git a/pkg/sources/circleci/circleci_test.go b/pkg/sources/circleci/circleci_test.go index 84927a993a5f..e8dc92f5a3ec 100644 --- a/pkg/sources/circleci/circleci_test.go +++ b/pkg/sources/circleci/circleci_test.go @@ -48,7 +48,7 @@ func TestSource_Scan(t *testing.T) { verify: true, }, wantErr: false, - totalScannedProjects: 2, + totalScannedProjects: 3, }, { name: "invalid token", @@ -108,6 +108,42 @@ func TestSource_Scan(t *testing.T) { } } +func TestRemoveCircleSha1Line(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "removes line containing CIRCLE_SHA1", + input: "export PATH=/usr/bin\nexport CIRCLE_SHA1=abcdef123456\necho done", + want: "export PATH=/usr/bin\necho done", + }, + { + name: "no CIRCLE_SHA1 line present", + input: "line one\nline two", + want: "line one\nline two", + }, + { + name: "removes multiple CIRCLE_SHA1 lines", + input: "a\nCIRCLE_SHA1=111\nb\nCIRCLE_SHA1=222\nc", + want: "a\nb\nc", + }, + { + name: "empty input", + input: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := removeCircleSha1Line([]byte(tt.input)) + assert.Equal(t, tt.want, string(got)) + }) + } +} + // additional test for edge cases func TestSource_EdgeCases(t *testing.T) { tests := []struct { diff --git a/pkg/sources/docker/README.md b/pkg/sources/docker/README.md index 862a93fe585d..79845ae2236c 100644 --- a/pkg/sources/docker/README.md +++ b/pkg/sources/docker/README.md @@ -59,88 +59,82 @@ The Docker source supports several image reference formats: For public images that don't require authentication: -**YAML Configuration:** -```yaml -sources: - - type: docker - name: public-images - docker: - unauthenticated: {} - images: - - nginx:latest - - alpine:3.18 -``` - **CLI Usage:** ```bash trufflehog docker --image nginx:latest ``` +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Docker + unauthenticated: {} + images: + - nginx:latest + - alpine:3.18 + name: public-images + type: SOURCE_TYPE_DOCKER + verify: true +``` + --- #### 2. Basic Authentication For private registries requiring username and password: +**CLI Usage:** + +Trufflehog does not provide basic authentication using username and password through CLI at the moment. + **YAML Configuration:** ```yaml sources: - - type: docker - name: private-registry - docker: - basic_auth: - username: myuser - password: mypassword - images: - - myregistry.com/private-image:latest - - myregistry.com/another-image:v1.0.0 +- connection: + '@type': type.googleapis.com/sources.Docker + basic_auth: + username: myuser + password: mypassword + images: + - myregistry.com/private-image:latest + - myregistry.com/another-image:v1.0.0 + name: private-registry + type: SOURCE_TYPE_DOCKER + verify: true ``` -**CLI Usage:** - -Trufflehog does not provide basic authentication using username and password through CLI at the moment. - --- #### 3. Bearer Token For registries using token-based authentication (e.g., Dockerhub registry): -**YAML Configuration:** -```yaml -sources: - - type: docker - name: truffle-packages - docker: - bearer_token: "ghp_xxxxxxxxxxxxxxxxxxxx" - images: - - myorg/myapp:latest - - myorg/frontend:v2.1.0 -``` - **CLI Usage:** ```bash trufflehog docker --image myorg/myapp:latest --bearer-token eyJ_xxxxxxxxxxxxxxxxxxxx ``` +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Docker + bearer_token: "ghp_xxxxxxxxxxxxxxxxxxxx" + images: + - myorg/myapp:latest + - myorg/frontend:v2.1.0 + name: truffle-packages + type: SOURCE_TYPE_DOCKER + verify: true +``` + --- #### 4. Docker Keychain Uses credentials from your local Docker configuration (`~/.docker/config.json`): -**YAML Configuration:** -```yaml -sources: - - type: docker - name: local-docker-creds - docker: - docker_keychain: true - images: - - myregistry.com/private-image:latest - - docker.io/myorg/app:latest -``` - **CLI Usage:** ```bash # First, authenticate with Docker @@ -161,6 +155,20 @@ docker login quay.io cat ~/.docker/config.json ``` +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Docker + docker_keychain: true + images: + - myregistry.com/private-image:latest + - docker.io/myorg/app:latest + name: local-docker-creds + type: SOURCE_TYPE_DOCKER + verify: true +``` + --- ### Namespace Scanning (This feature is currently in beta version and under testing) @@ -184,11 +192,13 @@ trufflehog docker --namespace myorg --registry-token **YAML Configuration:** ```yaml sources: - - type: docker - name: org-scan - docker: - namespace: myorg - registry_token: "ghp_xxxxxxxxxxxxxxxxxxxx" +- connection: + '@type': type.googleapis.com/sources.Docker + namespace: myorg + registry_token: "ghp_xxxxxxxxxxxxxxxxxxxx" + name: org-scan + type: SOURCE_TYPE_DOCKER + verify: true ``` Supported registries: @@ -281,19 +291,6 @@ docker login my-registry.io trufflehog docker --image my-registry.io/private-app:v1.0.0 ``` -## Testing Results - -| Test Case | Status | Command/Configuration | Registry URL | Notes | -|-----------|--------|----------------------|--------------|-------| -| Scan remote image on DockerHub | ✅ Success | `--image ` | https://hub.docker.com/ | Public images work without authentication | -| Scan specific tag of image on DockerHub | ✅ Success | `--image :` | https://hub.docker.com/ | Tag specification working correctly | -| Scan all images under namespace | In Progress | `--namespace ` | DockerHub, Quay, GHCR | Automatically discovers all public images | -| Scan remote image on Quay.io | ✅ Success | `--image quay.io/prometheus/prometheus` | https://quay.io/search | Public Quay.io registry supported | -| Scan multiple images | ✅ Success | `--image --image ` | Multiple registries | Sequential scanning of multiple images | -| Scan remote image on DockerHub with token | ✅ Success | `--token `(Generate token using username and password) | https://hub.docker.com/ | Authenticated scanning for private repos | -| Scan private image on Quay | ⏸️ Halted | N/A | https://quay.io/ | RedHat requires paid account for private repos | -| Scan private image on GHCR | ✅ Success | `--image ghcr.io/` | https://github.com/packages | GitHub Container Registry | - ## Troubleshooting ### Common Issues @@ -315,3 +312,8 @@ trufflehog docker --image my-registry.io/private-app:v1.0.0 **Issue**: Files not being scanned **Solution**: Check exclude patterns and file size limits. Verify files are under 50MB. + +--- + +**Issue**: Cannot scan private image on Quay.io +**Note**: Quay.io requires a paid account to access private repositories. (Not supported yet) diff --git a/pkg/sources/docker/docker.go b/pkg/sources/docker/docker.go index 135bb10c40a1..ea376610ac95 100644 --- a/pkg/sources/docker/docker.go +++ b/pkg/sources/docker/docker.go @@ -382,7 +382,7 @@ func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo image if err != nil { return err } - defer rc.Close() + defer func() { _ = rc.Close() }() // Configure parallel gzip decompression for better performance const ( @@ -394,7 +394,7 @@ func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo image if err != nil { return err } - defer gzipReader.Close() + defer func() { _ = gzipReader.Close() }() // Layers are tar archives, so we read them as tar files tarReader := tar.NewReader(gzipReader) diff --git a/pkg/sources/docker/docker_test.go b/pkg/sources/docker/docker_test.go index b8cfb689472c..bd8b026a5f6a 100644 --- a/pkg/sources/docker/docker_test.go +++ b/pkg/sources/docker/docker_test.go @@ -6,8 +6,7 @@ import ( "sync" "testing" - image "github.com/docker/docker/api/types/image" - dockerClient "github.com/docker/docker/client" + dockerClient "github.com/moby/moby/client" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/anypb" @@ -255,19 +254,19 @@ func TestDockerImageScanFromLocalDaemon(t *testing.T) { // pull the image here to ensure it exists locally img := "docker.io/trufflesecurity/secrets:latest" - client, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) + client, err := dockerClient.New(dockerClient.FromEnv) if err != nil { t.Errorf("Failed to create Docker client: %v", err) return } - resp, err := client.ImagePull(context.TODO(), img, image.PullOptions{}) + resp, err := client.ImagePull(context.TODO(), img, dockerClient.ImagePullOptions{}) if err != nil { t.Errorf("Failed to load image %s: %v", img, err) return } - defer resp.Close() + defer func() { _ = resp.Close() }() // if we don't read the response, the image will not be available in the local Docker daemon _, err = io.ReadAll(resp) diff --git a/pkg/sources/docker/registries_test.go b/pkg/sources/docker/registries_test.go index f6f3c04047dc..b661b80e934b 100644 --- a/pkg/sources/docker/registries_test.go +++ b/pkg/sources/docker/registries_test.go @@ -57,9 +57,10 @@ func TestGHCRListImages(t *testing.T) { ghcrImages, err := ghcr.ListImages(context.Background(), "ghcr.io/mongodb") assert.NoError(t, err) - assert.Equal(t, len(ghcrImages), 1) - - assert.Equal(t, ghcrImages, []string{"ghcr.io/mongodb/kingfisher"}) + // mongodb is a third-party namespace whose package list changes over time, + // so only assert on a known package rather than the exact set. + assert.NotEmpty(t, ghcrImages) + assert.Contains(t, ghcrImages, "ghcr.io/mongodb/kingfisher") } func TestDockerHubListImages_RateLimitError(t *testing.T) { diff --git a/pkg/sources/elasticsearch/README.md b/pkg/sources/elasticsearch/README.md new file mode 100644 index 000000000000..268571fa2a98 --- /dev/null +++ b/pkg/sources/elasticsearch/README.md @@ -0,0 +1,235 @@ +# Elasticsearch Source + +## Overview + +The Elasticsearch source lets TruffleHog scan documents stored in an Elasticsearch cluster for secrets, credentials, and sensitive data. It connects to the cluster, finds matching indices, and reads the documents inside them. + +## Elasticsearch Fundamentals + +### What is Elasticsearch? + +Elasticsearch is a search and analytics engine that stores data as documents inside indices. It is often used to store and search logs, so many logging tools send their log lines into Elasticsearch as documents. + +### Key Elasticsearch Terminology + +| Term | Description | +|------|-------------| +| **Node** | A single running instance of Elasticsearch. A cluster is made of one or more nodes | +| **Cluster** | A group of nodes working together and holding the same data | +| **Index** | A collection of documents, similar to a table in a normal database | +| **Document** | A single record stored inside an index, in JSON format | +| **Cloud ID** | An identifier used to connect to a managed Elastic Cloud deployment without listing node addresses | +| **API Key** | A credential used to authenticate API requests without a username and password | +| **Service Token** | A credential tied to a specific Elasticsearch service account | +| **Point in Time (PIT)** | A snapshot of the index state used so a search can page through results safely while data keeps changing | + +## Features + +- **Multiple Connection Methods**: Connect using node addresses or an Elastic Cloud ID +- **Multiple Authentication Methods**: Username and password, API key, or service token +- **Index Filtering**: Limit scanning to indices matching a pattern +- **Document Filtering**: Limit scanning to documents matching a query, or created after a timestamp +- **Concurrent Processing**: Documents are scanned in parallel across a number of workers +- **Continuous Scanning**: Optionally keep scanning a live cluster and pick up new documents as they arrive +- **Duplicate Avoidance**: Tracks the newest document timestamp per index so already seen documents are not scanned again on the next run. This tracking is kept in memory only, not in a database, so it only lasts for the current run (or the current best effort scan loop). Restarting TruffleHog loses this state, so use `--since-timestamp` again to avoid rescanning old documents. + +## Configuration + +### Connecting to a Cluster + +**CLI Usage:** + +You can connect using node addresses: + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 +``` + +Or using an Elastic Cloud ID: + +```bash +trufflehog elasticsearch --cloud-id "my-deployment:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==" +``` + +**YAML Configuration:** + +This is the correct shape for an Elasticsearch entry in a `--config` file: + +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Elasticsearch + nodes: + - "https://localhost:9200" + name: es-scan + type: SOURCE_TYPE_ELASTICSEARCH + verify: true +``` + +Note: as of this writing, `pkg/config/config.go` does not have a case for `SOURCE_TYPE_ELASTICSEARCH` in its source loader, so a config file entry like the one above will fail to load with `got unexpected source type`. Use the CLI flags or environment variables below until that is added. + +### Authentication Methods + +#### 1. Username and Password + +**CLI Usage:** +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --username elastic --password mypassword +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Elasticsearch + nodes: + - "https://localhost:9200" + username: elastic + password: mypassword + name: es-scan + type: SOURCE_TYPE_ELASTICSEARCH + verify: true +``` + +--- + +#### 2. API Key + +**CLI Usage:** +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --api-key xxxxxxxxxxxxxxxxxxxxxxxx +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Elasticsearch + nodes: + - "https://localhost:9200" + api_key: "xxxxxxxxxxxxxxxxxxxxxxxx" + name: es-scan + type: SOURCE_TYPE_ELASTICSEARCH + verify: true +``` + +--- + +#### 3. Service Token + +**CLI Usage:** +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --service-token AAEAAWVsYXN0aWMv... +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Elasticsearch + nodes: + - "https://localhost:9200" + service_token: "AAEAAWVsYXN0aWMv..." + name: es-scan + type: SOURCE_TYPE_ELASTICSEARCH + verify: true +``` + +Environment variables are also supported for every value above: `ELASTICSEARCH_NODES`, `ELASTICSEARCH_USERNAME`, `ELASTICSEARCH_PASSWORD`, `ELASTICSEARCH_CLOUD_ID`, `ELASTICSEARCH_API_KEY`, and `ELASTICSEARCH_SERVICE_TOKEN`. + +### Filtering What Gets Scanned + +**Index Pattern** + +Limit scanning to indices matching a pattern. The default is `*`, which matches every index. + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --index-pattern "logs-*" +``` + +**Query JSON** + +Provide a raw Elasticsearch query to filter which documents are scanned. + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --query-json '{"match": {"level": "error"}}' +``` + +**Since Timestamp** + +Only scan documents with a `@timestamp` field on or after the given time. This is ignored once TruffleHog has already recorded a newer timestamp from a previous scan of that index. + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --since-timestamp "2024-01-01T00:00:00Z" +``` + +### Best Effort Scan + +By default, TruffleHog scans the matching documents once and stops. Pass `--best-effort-scan` to keep the source running and repeatedly check the cluster for new documents. + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --best-effort-scan +``` + +## How Scanning Works + +### Scanning Process + +1. **Index Discovery**: Lists indices matching the configured index pattern +2. **Document Counting**: Counts matching documents in each index using the configured query and timestamp filters +3. **Work Distribution**: Splits the documents to scan into roughly equal groups across the configured number of workers +4. **Point in Time Search**: Each worker opens a point in time on its assigned index and pages through matching documents in order +5. **Document Scanning**: Each document's `message` field is sent to the detection engine as a chunk, tagged with the index name, document id, and timestamp +6. **Progress Tracking**: The newest document timestamp seen in each index is recorded, so a later run or a best effort scan loop can skip documents already scanned + +### What Gets Scanned + +- The `message` field of documents in indices matching the index pattern and query filters + +### What Doesn't Get Scanned + +- Documents outside the configured index pattern, query, or since timestamp filter +- Fields other than `@timestamp` and `message` + +## Usage Examples + +### Scanning a Local Cluster + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --username elastic --password mypassword +``` + +### Scanning an Elastic Cloud Deployment + +```bash +trufflehog elasticsearch --cloud-id "my-deployment:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==" --api-key xxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning Only Log Indices Created Since a Given Date + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --index-pattern "logs-*" --since-timestamp "2024-06-01T00:00:00Z" +``` + +### Continuously Scanning a Live Cluster + +```bash +trufflehog elasticsearch --nodes https://localhost:9200 --best-effort-scan +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Authentication failures when connecting to the cluster +**Solution**: Check that the username, password, API key, or service token is correct and has permission to read the target indices. + +--- + +**Issue**: No documents are scanned even though the cluster has data +**Solution**: Check the index pattern matches real index names, and that the query JSON or since timestamp filter is not excluding every document. + +--- + +**Issue**: The same documents keep getting scanned again on every run +**Solution**: This is expected the first time a since timestamp is used. On later runs TruffleHog uses the newest timestamp already seen, so the same documents are skipped unless best effort scan restarts from the same state. diff --git a/pkg/sources/elasticsearch/api.go b/pkg/sources/elasticsearch/api.go index 02a2480d8984..c75b283150f0 100644 --- a/pkg/sources/elasticsearch/api.go +++ b/pkg/sources/elasticsearch/api.go @@ -145,7 +145,7 @@ func makeElasticSearchRequest( return nil, err } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() rawData, err := io.ReadAll(res.Body) if err != nil { @@ -216,17 +216,17 @@ func fetchIndexDocumentCount( hits, ok := data["hits"].(map[string]any) if !ok { - return 0, errors.New("No hits in response") + return 0, errors.New("no hits in response") } total, ok := hits["total"].(map[string]any) if !ok { - return 0, errors.New("No total in hits") + return 0, errors.New("no total in hits") } count, ok := total["value"].(float64) if !ok { - return 0, errors.New("No value in total") + return 0, errors.New("no value in total") } return int(count), nil @@ -249,7 +249,7 @@ func createPITForSearch( pitID, ok := data["id"].(string) if !ok { - return "", errors.New("No id in response") + return "", errors.New("no id in response") } return pitID, nil @@ -321,7 +321,7 @@ func processSearchedDocuments( if !ok { apiErr, ok := searchResults["error"].(map[string]any) if ok { - return 0, fmt.Errorf("Error fetching search results: %v\n", apiErr) + return 0, fmt.Errorf("error fetching search results: %v", apiErr) } continue } diff --git a/pkg/sources/elasticsearch/api_test.go b/pkg/sources/elasticsearch/api_test.go new file mode 100644 index 000000000000..1d1c7847b956 --- /dev/null +++ b/pkg/sources/elasticsearch/api_test.go @@ -0,0 +1,102 @@ +package elasticsearch + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestFilterParams_Query(t *testing.T) { + t.Run("no query JSON and no timestamp produces an empty query", func(t *testing.T) { + fp := FilterParams{} + + query, err := fp.Query(time.Time{}) + assert.NoError(t, err) + assert.Equal(t, map[string]any{}, query["query"]) + }) + + t.Run("invalid query JSON returns an error", func(t *testing.T) { + fp := FilterParams{queryJSON: "not json"} + + _, err := fp.Query(time.Time{}) + assert.Error(t, err) + }) + + t.Run("latest timestamp adds a range clause on @timestamp", func(t *testing.T) { + fp := FilterParams{} + latest := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC) + + query, err := fp.Query(latest) + assert.NoError(t, err) + + clause, ok := query["query"].(map[string]any) + assert.True(t, ok) + + rangeClause, ok := clause["range"].(map[string]map[string]string) + assert.True(t, ok) + assert.Equal(t, latest.Format(time.RFC3339), rangeClause["@timestamp"]["gte"]) + }) + + t.Run("since timestamp is used when latest timestamp is zero", func(t *testing.T) { + fp := FilterParams{sinceTimestamp: "2024-01-01T00:00:00Z"} + + query, err := fp.Query(time.Time{}) + assert.NoError(t, err) + + clause, ok := query["query"].(map[string]any) + assert.True(t, ok) + + rangeClause, ok := clause["range"].(map[string]map[string]string) + assert.True(t, ok) + assert.Equal(t, "2024-01-01T00:00:00Z", rangeClause["@timestamp"]["gte"]) + }) + + t.Run("latest timestamp takes priority over since timestamp", func(t *testing.T) { + fp := FilterParams{sinceTimestamp: "2024-01-01T00:00:00Z"} + latest := time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC) + + query, err := fp.Query(latest) + assert.NoError(t, err) + + clause := query["query"].(map[string]any) + rangeClause := clause["range"].(map[string]map[string]string) + assert.Equal(t, latest.Format(time.RFC3339), rangeClause["@timestamp"]["gte"]) + }) +} + +func TestIndex_DocumentAlreadySeen(t *testing.T) { + index := NewIndex() + + firstTimestamp := "2024-01-01T00:00:00Z" + + // A document with a newer timestamp than the zero value is treated as new, + // and becomes the index's latest timestamp. It is not recorded in the + // seen list, since the list was just reset. + seen := index.DocumentAlreadySeen(&Document{id: "doc-1", timestamp: firstTimestamp}) + assert.False(t, seen) + + // A second document at the same (not newer) timestamp is treated as new, + // since the run has not been marked complete yet, and it is recorded in + // the seen list. + seen = index.DocumentAlreadySeen(&Document{id: "doc-2", timestamp: firstTimestamp}) + assert.False(t, seen) + + // Once the run completes, the latest timestamp is recorded as the + // timestamp of the last run. + index.UpdateLatestTimestampLastRun() + + // A document already recorded in the seen list at that timestamp is now + // reported as a duplicate. + seen = index.DocumentAlreadySeen(&Document{id: "doc-2", timestamp: firstTimestamp}) + assert.True(t, seen) + + // A new document ID at that same timestamp is still treated as new. + seen = index.DocumentAlreadySeen(&Document{id: "doc-3", timestamp: firstTimestamp}) + assert.False(t, seen) + + // A document with an unparsable timestamp is treated as new and does not + // change index state. + seen = index.DocumentAlreadySeen(&Document{id: "doc-4", timestamp: "not-a-timestamp"}) + assert.False(t, seen) +} diff --git a/pkg/sources/elasticsearch/elasticsearch_integration_test.go b/pkg/sources/elasticsearch/elasticsearch_integration_test.go index cc7b821d084f..4062f79c6fbf 100644 --- a/pkg/sources/elasticsearch/elasticsearch_integration_test.go +++ b/pkg/sources/elasticsearch/elasticsearch_integration_test.go @@ -86,7 +86,7 @@ func TestSource_ElasticAPI(t *testing.T) { if err != nil { t.Fatal(err) } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() t.Run( "Adding a document to a new index creates a single index", @@ -126,7 +126,7 @@ func TestSource_ElasticAPI(t *testing.T) { if err != nil { t.Fatal(err) } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() t.Run( "Indices have the correct document count", @@ -349,7 +349,7 @@ func TestSource_ElasticAPI(t *testing.T) { if err != nil { t.Fatal(err) } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() } docSearch := DocumentSearch{ diff --git a/pkg/sources/errors.go b/pkg/sources/errors.go index 3933a7f0ba68..fc52a4b691cb 100644 --- a/pkg/sources/errors.go +++ b/pkg/sources/errors.go @@ -74,3 +74,22 @@ func (t TargetedScanError) Error() string { func (t TargetedScanError) Unwrap() error { return t.Err } + +// TargetNotFoundError indicates a targeted scan failed because the requested +// target is definitively absent from the source (e.g. the file, commit, or +// message it pointed to was deleted), as opposed to inaccessible due to +// authorization or transient failures. Consumers can treat this as terminal: +// retrying cannot succeed until a fresh scan updates the stored location. +type TargetNotFoundError struct { + Err error +} + +var _ error = (*TargetNotFoundError)(nil) + +func (t TargetNotFoundError) Error() string { + return t.Err.Error() +} + +func (t TargetNotFoundError) Unwrap() error { + return t.Err +} diff --git a/pkg/sources/errors_test.go b/pkg/sources/errors_test.go index e7b9d3a035a3..ee122d74eb91 100644 --- a/pkg/sources/errors_test.go +++ b/pkg/sources/errors_test.go @@ -7,7 +7,7 @@ import ( "testing" ) -var testError = fmt.Errorf("simulated failure") +var errTest = fmt.Errorf("simulated failure") func TestNewScanErrors(t *testing.T) { testCases := []struct { @@ -86,7 +86,7 @@ func TestScanErrorsAdd(t *testing.T) { wg.Add(1) go func() { for j := 0; j < tc.wantErr/tc.concurrency; j++ { - se.Add(testError) + se.Add(errTest) } wg.Done() }() @@ -137,7 +137,7 @@ func TestScanErrorsCount(t *testing.T) { wg.Add(1) go func() { for j := 0; j < tc.wantErrCnt/tc.concurrency; j++ { - se.Add(testError) + se.Add(errTest) } wg.Done() }() @@ -153,9 +153,25 @@ func TestScanErrorsCount(t *testing.T) { func TestScanErrorsString(t *testing.T) { se := NewScanErrors() - se.Add(testError) - want := `["` + testError.Error() + `"]` + se.Add(errTest) + want := `["` + errTest.Error() + `"]` if got := fmt.Sprintf("%v", se); got != want { t.Errorf("got %q, want %q", got, want) } } + +func TestTargetNotFoundErrorUnwrapsThroughJobChain(t *testing.T) { + inner := &TargetNotFoundError{Err: fmt.Errorf("could not download file for scan: 404")} + // The chain a targeted scan failure travels before a consumer sees it: + // scanTargets wraps in TargetedScanError, the source manager wraps in + // Fatal, and JobProgressMetrics.FatalErrors joins the results. + chain := errors.Join(Fatal{&TargetedScanError{Err: inner, SecretID: 42}}) + + var tnf *TargetNotFoundError + if !errors.As(chain, &tnf) { + t.Fatal("TargetNotFoundError not found in wrapped chain") + } + if tnf.Error() != inner.Error() { + t.Fatalf("unexpected message: %q", tnf.Error()) + } +} diff --git a/pkg/sources/filesystem/README.md b/pkg/sources/filesystem/README.md new file mode 100644 index 000000000000..1dd67ef37c37 --- /dev/null +++ b/pkg/sources/filesystem/README.md @@ -0,0 +1,210 @@ +# Filesystem Source + +## Overview + +The Filesystem source lets TruffleHog scan files and directories on local disk for secrets, credentials, and sensitive data. It can scan a single file, walk a whole directory tree, or follow symlinks to files and directories outside the given path. + +## Filesystem Fundamentals + +### What does this source scan? + +This source scans paths on the machine running TruffleHog. A path can point to a single file, a directory, or a symlink. When a directory is given, TruffleHog walks every file inside it, including files in subdirectories. + +### Key Terminology + +| Term | Description | +|------|-------------| +| **Path** | A file or directory location to scan. You can list more than one | +| **Symlink** | A file that points to another file or directory instead of holding data itself | +| **Symlink Depth** | How many symlinks in a row TruffleHog will follow before it stops, to avoid following a loop forever | +| **Include Paths File** | A file with a list of patterns. Only files matching one of these patterns are scanned | +| **Exclude Paths File** | A file with a list of patterns. Files matching one of these patterns are skipped | +| **Binary File** | A file that is not plain text, such as an image or a compiled program | +| **Resume Info** | Progress information saved during a scan so a long scan can continue where it left off after a restart | + +## Features + +- **File and Directory Scanning**: Scan a single file or an entire directory tree +- **Symlink Support**: Optionally follow symlinks to files and directories, up to a configured depth +- **Path Filtering**: Include or exclude files using pattern files +- **Binary File Skipping**: Optionally skip files that are not plain text +- **Concurrent Processing**: Files within a directory are scanned in parallel +- **Resumable Scans**: A scan that stops partway through a large directory can pick up from where it stopped + +## Configuration + +### Scanning Paths + +**CLI Usage:** +```bash +trufflehog filesystem /var/log /home/user/project/config.yaml +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Filesystem + paths: + - /var/log + - /home/user/project/config.yaml + name: local-scan + type: SOURCE_TYPE_FILESYSTEM + verify: true +``` + +You can also pass directories using the `--directory` flag, which can be repeated: + +```bash +trufflehog filesystem --directory /var/log --directory /etc +``` + +### Following Symlinks + +By default, symlinks are not followed. Set a maximum depth to allow TruffleHog to follow them. + +**CLI Usage:** +```bash +trufflehog filesystem /home/user/project --max-symlink-depth 5 +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Filesystem + paths: + - /home/user/project + max_symlink_depth: 5 + name: local-scan + type: SOURCE_TYPE_FILESYSTEM + verify: true +``` + +The highest depth allowed is 40. Setting a higher number will cause the scan to fail with an error at startup. + +### Skipping Binary Files + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Filesystem + paths: + - /home/user/project + skip_binaries: true + name: local-scan + type: SOURCE_TYPE_FILESYSTEM + verify: true +``` + +### Path Filtering + +Use plain text files with one pattern per line to include or exclude files from a scan. + +**CLI Usage:** +```bash +trufflehog filesystem /home/user/project --include-paths /home/user/include-patterns.txt --exclude-paths /home/user/exclude-patterns.txt +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Filesystem + paths: + - /home/user/project + include_paths_file: /home/user/include-patterns.txt + exclude_paths_file: /home/user/exclude-patterns.txt + name: local-scan + type: SOURCE_TYPE_FILESYSTEM + verify: true +``` + +## How Scanning Works + +### Scanning Process + +1. **Path Check**: For each configured path, TruffleHog checks whether it is a file, a directory, or a symlink +2. **Directory Walk**: If it is a directory, every entry inside is visited. Subdirectories are visited the same way, going deeper until every file is reached +3. **Symlink Handling**: If an entry is a symlink and following symlinks is turned on, TruffleHog follows it to the file or directory it points to, up to the configured depth +4. **File Filtering**: Each file is checked against the include and exclude patterns before it is scanned +5. **Binary Check**: If skipping binaries is turned on, files that look like binary files are skipped +6. **Content Scanning**: The file is opened and its content is sent to the detection engine as chunks +7. **Progress Tracking**: The path of the last file scanned in each directory is saved, so a scan that stops can resume from that point + +### What Gets Scanned + +- Regular files at the given paths +- Files inside directories at the given paths, including subdirectories +- Files reached by following a symlink, when symlink following is turned on + +### What Doesn't Get Scanned + +- Symlinks, when symlink following is turned off +- Files that go past the configured maximum symlink depth +- Files matching an exclude pattern, or not matching an include pattern, when pattern files are set +- Binary files, when skipping binaries is turned on +- Files that are not regular files, such as device files or sockets + +## Usage Examples + +### Scanning a Single File + +```bash +trufflehog filesystem /home/user/project/config.yaml +``` + +### Scanning a Directory + +```bash +trufflehog filesystem /home/user/project +``` + +### Scanning Multiple Paths + +```bash +trufflehog filesystem /home/user/project /var/log/app.log +``` + +### Scanning a Directory and Following Symlinks + +```bash +trufflehog filesystem /home/user/project --max-symlink-depth 10 +``` + +### Scanning While Skipping Binary Files + +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Filesystem + paths: + - /home/user/project + skip_binaries: true + name: local-scan + type: SOURCE_TYPE_FILESYSTEM + verify: true +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Symlinks are not being scanned +**Solution**: Set `max_symlink_depth` (or `--max-symlink-depth`) to a value greater than zero. Symlinks are skipped by default. + +--- + +**Issue**: `specified symlink depth exceeds the allowed max` +**Solution**: The maximum allowed symlink depth is 40. Use a smaller value. + +--- + +**Issue**: Some files are missing from scan results +**Solution**: Check the include and exclude pattern files for a pattern that matches those files, and confirm `skip_binaries` is not turned on for text files that look like binary data. + +--- + +**Issue**: A restarted scan seems to repeat or skip files +**Solution**: Resume tracking saves the last file scanned inside each directory. If the directory contents changed between runs, the resume point may no longer line up with the same files. diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 5aced6666eb0..9e6206423eb2 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "github.com/go-errors/errors" @@ -41,9 +42,11 @@ type Source struct { } // Ensure the Source satisfies the interfaces at compile time -var _ sources.Source = (*Source)(nil) -var _ sources.SourceUnitUnmarshaller = (*Source)(nil) -var _ sources.SourceUnitEnumChunker = (*Source)(nil) +var ( + _ sources.Source = (*Source)(nil) + _ sources.SourceUnitUnmarshaller = (*Source)(nil) + _ sources.SourceUnitEnumChunker = (*Source)(nil) +) // max symlink depth allowed const defaultMaxSymlinkDepth = 40 @@ -115,7 +118,7 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ if common.IsDone(ctx) { return nil } - s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", rootPath), "") + s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", rootPath), s.GetProgress().EncodedResumeInfo) cleanPath := filepath.Clean(rootPath) fileInfo, err := os.Lstat(cleanPath) @@ -229,6 +232,25 @@ func (s *Source) scanSymlink( return nil } +// comparePathsForResume orders two cleaned paths the way a depth-first walk over +// os.ReadDir-sorted entries does: component by component. A raw string comparison +// is wrong here because the separator '/' (0x2F) sorts after characters that are +// valid in a path component, such as '-' (0x2D) or '.' (0x2E). For example, as raw +// strings "/root/blue-team-deprecated" < "/root/blue-team/file.txt" (the '-' after +// "blue-team" sorts before '/'), which would make scanDir wrongly skip the sibling +// directory "blue-team-deprecated" when resuming inside "blue-team". Comparing per +// component matches os.ReadDir's ordering, where "blue-team" sorts before +// "blue-team-deprecated". Returns -1, 0, or 1. +func comparePathsForResume(a, b string) int { + // slices.Compare walks the components lexicographically and, when one path is + // a prefix of the other, orders the shorter (ancestor) path first — matching + // the depth-first walk order described above. + return slices.Compare( + strings.Split(a, string(filepath.Separator)), + strings.Split(b, string(filepath.Separator)), + ) +} + func (s *Source) scanDir( ctx trContext.Context, chunksChan chan *sources.Chunk, @@ -258,8 +280,8 @@ func (s *Source) scanDir( if resumeAfter != "" && !strings.HasPrefix(resumeAfter, path+string(filepath.Separator)) && resumeAfter != path { // Resume point is not in this subtree. Compare paths to determine if we // should skip this directory (already scanned) or process it (already passed). - if path < resumeAfter { - // This directory comes before the resume point lexicographically, + if comparePathsForResume(path, resumeAfter) < 0 { + // This directory comes before the resume point in traversal order, // meaning it was already fully scanned. Skip it entirely. return nil } @@ -362,7 +384,7 @@ func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, if err != nil { return fmt.Errorf("unable to open file: %w", err) } - defer inputFile.Close() + defer func() { _ = inputFile.Close() }() fileCtx.Logger().V(3).Info("scanning file") diff --git a/pkg/sources/filesystem/filesystem_resume_test.go b/pkg/sources/filesystem/filesystem_resume_test.go new file mode 100644 index 000000000000..d39be148124b --- /dev/null +++ b/pkg/sources/filesystem/filesystem_resume_test.go @@ -0,0 +1,95 @@ +package filesystem + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// TestResumptionInChunksAfterRestart guards against a regression where a +// filesystem scan restarts from the beginning after the scanner process is +// restarted (for example by a version rollout). +// +// This exercises the legacy ENUMERATE_AND_SCAN path (Chunks), and it restores +// the resume point the same way the scanner does after reclaiming an +// in-progress job: by assigning the encoded resume info to the top-level +// Progress.EncodedResumeInfo string, WITHOUT calling SetEncodedResumeInfoFor. +// +// Both of those details matter, and are why the other resumption tests in this +// package do not catch the bug: +// +// - They call ChunkUnit, which walks straight into scanDir. Chunks first runs +// s.SetProgressComplete(i, len(s.paths), "...", "") for each path, and +// SetProgressComplete unconditionally overwrites EncodedResumeInfo with its +// (empty) fourth argument. +// - They set the resume point via SetEncodedResumeInfoFor, which hydrates the +// internal encodedResumeInfoByID map. Once that map is non-nil, the empty +// SetProgressComplete write no longer matters, because GetEncodedResumeInfoFor +// reads the map, not the string. +// +// On a freshly started process the map is still nil, so the empty write destroys +// the restored resume point before scanDir ever reads it, and the scan starts +// over from the first (lowest-sorted) entry. +func TestResumptionInChunksAfterRestart(t *testing.T) { + ctx := trContext.Background() + + // Top-level directories that sort deterministically, mirroring the sorted + // S3-prefix layout the customer scans. + rootDir, err := os.MkdirTemp("", "trufflehog-resumption-chunks-test") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(rootDir) }) + + dirs := []string{"aaa", "bbb", "ccc", "ddd"} + filesByDir := make(map[string]string, len(dirs)) + for _, d := range dirs { + dirPath := filepath.Join(rootDir, d) + require.NoError(t, os.Mkdir(dirPath, 0755)) + filePath := filepath.Join(dirPath, "file.txt") + require.NoError(t, os.WriteFile(filePath, []byte("content of "+d), 0644)) + filesByDir[d] = filePath + } + + conn, err := anypb.New(&sourcespb.Filesystem{Paths: []string{rootDir}}) + require.NoError(t, err) + + s := Source{} + // concurrency 1 keeps the walk deterministic and isolates this test from the + // separate out-of-order resume-write behavior of scanDir's worker pool. + require.NoError(t, s.Init(ctx, "test resumption chunks", 0, 0, true, conn, 1)) + + // Simulate a job restored after already scanning through bbb/file.txt. + // aaa and bbb must be skipped; ccc and ddd must be scanned. + resumePoint := filesByDir["bbb"] + encoded, err := json.Marshal(map[string]string{rootDir: resumePoint}) + require.NoError(t, err) + s.GetProgress().EncodedResumeInfo = string(encoded) + + chunksCh := make(chan *sources.Chunk, len(dirs)) + go func() { + defer close(chunksCh) + assert.NoError(t, s.Chunks(ctx, chunksCh)) + }() + + scannedFiles := make(map[string]bool) + for chunk := range chunksCh { + scannedFiles[chunk.SourceMetadata.GetFilesystem().GetFile()] = true + } + + assert.False(t, scannedFiles[filesByDir["aaa"]], + "aaa/file.txt should have been skipped (before resume point); the scan restarted from the beginning") + assert.False(t, scannedFiles[filesByDir["bbb"]], + "bbb/file.txt should have been skipped (the resume point itself)") + assert.True(t, scannedFiles[filesByDir["ccc"]], + "ccc/file.txt should have been scanned (after resume point)") + assert.True(t, scannedFiles[filesByDir["ddd"]], + "ddd/file.txt should have been scanned (after resume point)") +} diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go index 5a964d6714b6..67b5e0c4cbf9 100644 --- a/pkg/sources/filesystem/filesystem_test.go +++ b/pkg/sources/filesystem/filesystem_test.go @@ -132,7 +132,7 @@ func TestScanFile(t *testing.T) { func TestScanBinaryFile(t *testing.T) { tmpfile, err := os.CreateTemp("", "example.bin") require.NoError(t, err) - defer os.Remove(tmpfile.Name()) + defer func() { _ = os.Remove(tmpfile.Name()) }() // binary data that decodes to "TuffleHog" fileContents := []byte{0x54, 0x75, 0x66, 0x66, 0x6C, 0x65, 0x48, 0x6F, 0x67} @@ -171,7 +171,7 @@ func TestEnumerate(t *testing.T) { // Setup the connection to test enumeration. dir, err := os.MkdirTemp("", "trufflehog-test-enumerate") assert.NoError(t, err) - defer os.RemoveAll(dir) + defer func() { _ = os.RemoveAll(dir) }() units := []string{ "/one", "/two", "/three", @@ -184,14 +184,14 @@ func TestEnumerate(t *testing.T) { if i < 3 { f, err := os.Create(fullPath) assert.NoError(t, err) - f.Close() + _ = f.Close() } else { assert.NoError(t, os.MkdirAll(fullPath, 0755)) // Create a file in the directory for enumeration to find. f, err := os.CreateTemp(fullPath, "file") assert.NoError(t, err) units[i] = f.Name() - f.Close() + _ = f.Close() } } conn, err := anypb.New(&sourcespb.Filesystem{ @@ -309,7 +309,7 @@ func TestChunkUnitReporterErr(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(tmpfile.Name()) + defer func() { _ = os.Remove(tmpfile.Name()) }() fileContents := []byte("TestChunkUnit") _, err = tmpfile.Write(fileContents) @@ -383,7 +383,7 @@ func TestScanSubDirFile(t *testing.T) { testDir := filepath.Join(os.TempDir(), "trufflehog-test") err := os.MkdirAll(testDir, 0755) require.NoError(t, err) - defer os.RemoveAll(testDir) + defer func() { _ = os.RemoveAll(testDir) }() // Create a subdirectory and file childDir := filepath.Join(testDir, "child") @@ -422,7 +422,7 @@ func TestSkipBinaries(t *testing.T) { // Create a temporary directory for testing tempDir, err := os.MkdirTemp("", "trufflehog_test") require.NoError(t, err) - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() // Create a binary file (executable) binaryFile := filepath.Join(tempDir, "test.exe") @@ -679,6 +679,111 @@ func TestResumptionWithNestedDirectories(t *testing.T) { assert.Equal(t, 1, len(reporter.Chunks), "expected exactly 1 file to be scanned") } +func TestResumptionWithPrefixSiblingDirectories(t *testing.T) { + ctx := trContext.Background() + + // Create sibling directories where one name is a prefix of the other: + // root/ + // blue-team/ + // project-notes.txt + // blue-team-deprecated/ + // AWSCredentials.txt + // + // os.ReadDir sorts "blue-team" before "blue-team-deprecated", so a scan that + // resumes inside blue-team must still descend into blue-team-deprecated. A raw + // string comparison of the directory path against the resume point would skip + // blue-team-deprecated, because '-' (0x2D) sorts before the separator '/' (0x2F). + rootDir, err := os.MkdirTemp("", "trufflehog-resumption-prefix-test") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(rootDir) }) + + dirs := map[string]string{ + "blue-team": "project-notes.txt", + "blue-team-deprecated": "AWSCredentials.txt", + } + for dir, file := range dirs { + dirPath := filepath.Join(rootDir, dir) + require.NoError(t, os.Mkdir(dirPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dirPath, file), []byte("content of "+file), 0644)) + } + + conn, err := anypb.New(&sourcespb.Filesystem{}) + require.NoError(t, err) + + s := Source{} + err = s.Init(ctx, "test resumption prefix sibling", 0, 0, true, conn, 1) + require.NoError(t, err) + + // Resume inside blue-team. project-notes.txt is the resume point (already + // scanned); AWSCredentials.txt in the sibling blue-team-deprecated comes after + // it in traversal order and must still be scanned. + resumePoint := filepath.Join(rootDir, "blue-team", "project-notes.txt") + s.SetEncodedResumeInfoFor(rootDir, resumePoint) + + reporter := sourcestest.TestReporter{} + err = s.ChunkUnit(ctx, sources.CommonSourceUnit{ID: rootDir}, &reporter) + require.NoError(t, err) + + scannedFiles := make(map[string]bool) + for _, chunk := range reporter.Chunks { + scannedFiles[filepath.Base(chunk.SourceMetadata.GetFilesystem().GetFile())] = true + } + + assert.False(t, scannedFiles["project-notes.txt"], "project-notes.txt should have been skipped (the resume point itself)") + assert.True(t, scannedFiles["AWSCredentials.txt"], "AWSCredentials.txt should have been scanned (sibling dir after the resume point)") + assert.Equal(t, 1, len(reporter.Chunks), "expected exactly 1 file to be scanned") +} + +func TestComparePathsForResume(t *testing.T) { + sep := string(filepath.Separator) + p := func(parts ...string) string { return sep + filepath.Join(parts...) } + + tests := []struct { + name string + a string + b string + want int + }{ + {"equal", p("root", "aaa"), p("root", "aaa"), 0}, + {"before sibling", p("root", "aaa"), p("root", "bbb", "file.txt"), -1}, + {"after sibling", p("root", "ccc"), p("root", "bbb", "file.txt"), 1}, + {"ancestor before descendant", p("root", "aaa"), p("root", "aaa", "file.txt"), -1}, + // Prefix-sibling: "blue-team-deprecated" must sort AFTER a resume point + // inside "blue-team", matching os.ReadDir order (regression for the raw + // string comparison where '-' < '/'). + {"prefix sibling sorts after", p("root", "blue-team-deprecated"), p("root", "blue-team", "notes.txt"), 1}, + {"prefix sibling reverse", p("root", "blue-team", "notes.txt"), p("root", "blue-team-deprecated"), -1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, comparePathsForResume(tt.a, tt.b)) + }) + } +} + +func TestInit_MaxSymlinkDepthWithinLimit(t *testing.T) { + ctx := trContext.Background() + + conn, err := anypb.New(&sourcespb.Filesystem{MaxSymlinkDepth: defaultMaxSymlinkDepth}) + require.NoError(t, err) + + s := Source{} + err = s.Init(ctx, "test max symlink depth within limit", 0, 0, true, conn, 1) + require.NoError(t, err) + assert.Equal(t, defaultMaxSymlinkDepth, s.maxSymlinkDepth) +} + +func TestInit_MaxSymlinkDepthExceedsLimit(t *testing.T) { + ctx := trContext.Background() + + conn, err := anypb.New(&sourcespb.Filesystem{MaxSymlinkDepth: defaultMaxSymlinkDepth + 1}) + require.NoError(t, err) + + s := Source{} + err = s.Init(ctx, "test max symlink depth exceeds limit", 0, 0, true, conn, 1) + assert.Error(t, err) +} + func TestResumptionWithOutOfSubtreeResumePoint(t *testing.T) { ctx := trContext.Background() diff --git a/pkg/sources/gcs/README.md b/pkg/sources/gcs/README.md new file mode 100644 index 000000000000..f32ddaede943 --- /dev/null +++ b/pkg/sources/gcs/README.md @@ -0,0 +1,277 @@ +# GCS Source + +## Overview + +The GCS source lets TruffleHog scan files stored in Google Cloud Storage buckets for secrets, credentials, and sensitive data. It lists the buckets in a project, walks the objects inside them, and scans each object's contents. + +## GCS Fundamentals + +### What is GCS? + +Google Cloud Storage (GCS) is Google's object storage service. Files are stored as objects inside buckets, and buckets belong to a Google Cloud project. + +### Key GCS Terminology + +| Term | Description | +|------|-------------| +| **Project** | A Google Cloud project that owns one or more buckets | +| **Bucket** | A container that holds objects, similar to a top level folder | +| **Object** | A single file stored inside a bucket | +| **Service Account** | A Google Cloud identity used by an application to authenticate, instead of a person | +| **Application Default Credentials (ADC)** | Credentials that Google Cloud client libraries pick up automatically from the environment, such as a service account attached to a VM or Cloud Run service | +| **ACL** | Access Control List. Defines who can read or write an object or bucket | + +## Features + +- **Multiple Authentication Methods**: Service account file, service account JSON, API key, OAuth, Application Default Credentials, or unauthenticated for public buckets +- **Bucket and Object Filtering**: Scan only specific buckets or objects, or exclude specific ones, using glob patterns +- **Size Limits**: Skips objects larger than a configurable maximum size, so scans do not stall on very large files +- **Concurrent Processing**: Buckets and objects are processed in parallel across a configurable number of workers +- **Resumable Scans**: Tracks which objects have already been scanned so a scan can be resumed without redoing work + +## Configuration + +### Authentication Methods + +#### 1. Application Default Credentials (ADC) + +Uses the credentials Google Cloud already provides in the environment, such as a service account attached to a VM. + +**CLI Usage:** +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GCS + project_id: my-gcp-project + adc: {} + name: gcs-scan + type: SOURCE_TYPE_GCS + verify: true +``` + +--- + +#### 2. Service Account File + +**CLI Usage:** +```bash +trufflehog gcs --project-id my-gcp-project --service-account /path/to/service-account.json +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GCS + project_id: my-gcp-project + service_account_file: /path/to/service-account.json + name: gcs-scan + type: SOURCE_TYPE_GCS + verify: true +``` + +--- + +#### 3. API Key + +Can only be used for public buckets. + +**CLI Usage:** +```bash +trufflehog gcs --project-id my-gcp-project --api-key AIzaxxxxxxxxxxxxxxxxxxxxxxxx +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GCS + project_id: my-gcp-project + api_key: "AIzaxxxxxxxxxxxxxxxxxxxxxxxx" + name: gcs-scan + type: SOURCE_TYPE_GCS + verify: true +``` + +--- + +#### 4. JSON Service Account + +Same as the service account file method, but the JSON content is passed directly instead of a file path. This method is only available in the YAML config, not the CLI. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GCS + project_id: my-gcp-project + json_service_account: '{"type": "service_account", "project_id": "my-gcp-project", ...}' + name: gcs-scan + type: SOURCE_TYPE_GCS + verify: true +``` + +--- + +#### 5. OAuth + +Uses an OAuth2 client ID with a refresh token and access token. This method is only available in the YAML config, not the CLI. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GCS + project_id: my-gcp-project + oauth: + client_id: xxxxxxxxxxxx.apps.googleusercontent.com + refresh_token: "1//xxxxxxxxxxxxxxxxxxxxxxxxxxxx" + access_token: "ya29.xxxxxxxxxxxxxxxxxxxxxxxx" + name: gcs-scan + type: SOURCE_TYPE_GCS + verify: true +``` + +--- + +#### 6. Unauthenticated + +Can only be used for public buckets. The project ID is not needed, but the buckets to scan must be listed with `include_buckets`, since the source cannot list all buckets in a project without authentication. + +**CLI Usage:** +```bash +trufflehog gcs --without-auth --include-buckets my-public-bucket +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GCS + unauthenticated: {} + include_buckets: + - my-public-bucket + name: gcs-scan + type: SOURCE_TYPE_GCS + verify: true +``` + +Environment variables are also supported: `GOOGLE_CLOUD_PROJECT` for the project ID, and `GOOGLE_API_KEY` for the API key. + +### Filtering What Gets Scanned + +**Include or Exclude Buckets** + +Only one of include or exclude can be used at a time for buckets. If both are set, include wins. + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --include-buckets "logs-*" +``` + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --exclude-buckets "temp-*" +``` + +**Include or Exclude Objects** + +Same rule applies, only one of include or exclude can be used at a time for objects. If a bucket is already known, it is faster to also set `include_buckets` alongside `include_objects`. + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --include-objects "*.env" +``` + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --exclude-objects "*.log" +``` + +**Maximum Object Size** + +Objects larger than this size are skipped. The default is 10MB. Setting it to 0, a negative value, or anything above 50MB resets it back to the 10MB default instead of using the value given. + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --max-object-size 20MB +``` + +**Number of Workers** + +There is no GCS specific flag for this. Use TruffleHog's global `--concurrency` flag to set how many buckets and objects are processed at the same time. If not set, it defaults to the number of CPU cores available. + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --concurrency 10 +``` + +## How Scanning Works + +### Scanning Process + +1. **Enumeration**: Lists every bucket in the project (or uses the buckets given with `include_buckets` when unauthenticated), then counts the objects in each one. This count is used to show scan progress. +2. **Bucket Listing**: Lists the buckets again to start the actual scan, applying the include and exclude filters. +3. **Object Listing**: For each bucket, lists its objects in parallel, applying the include and exclude filters and skipping objects over the size limit. +4. **Object Scanning**: Downloads each object and sends its contents to the detection engine as a chunk, tagged with the bucket name, object name, and other metadata. +5. **Progress Tracking**: Keeps a cache of the MD5 hash of every object already scanned, so if the same scan is resumed later, already scanned objects are skipped. + +### What Gets Scanned + +- Objects in buckets matching the include and exclude bucket filters +- Objects matching the include and exclude object filters +- Objects at or under the configured maximum object size + +### What Doesn't Get Scanned + +- Objects excluded by the include or exclude bucket or object filters +- Objects larger than the maximum object size +- Objects with file extensions TruffleHog already knows to skip, such as common image, audio, video, and font file types +- Empty objects, since they have a size of 0 + +## Usage Examples + +### Scanning All Buckets in a Project with ADC + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment +``` + +### Scanning a Public Bucket Without Authentication + +```bash +trufflehog gcs --without-auth --include-buckets my-public-bucket +``` + +### Scanning Only Specific Buckets + +```bash +trufflehog gcs --project-id my-gcp-project --cloud-environment --include-buckets "prod-logs,prod-backups" +``` + +### Scanning with a Service Account and a Size Limit + +```bash +trufflehog gcs --project-id my-gcp-project --service-account /path/to/service-account.json --max-object-size 5MB +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Authentication failures when connecting to GCS +**Solution**: Check that the service account, API key, or ADC setup is correct and has permission to read the target buckets. Only one authentication method can be used at a time. + +--- + +**Issue**: `project ID is required` error +**Solution**: A project ID is required for every authentication method except unauthenticated scanning. Pass `--project-id` or set the `GOOGLE_CLOUD_PROJECT` environment variable. + +--- + +**Issue**: No buckets found when scanning without authentication +**Solution**: Unauthenticated scans cannot list buckets in a project. Pass the exact bucket names with `--include-buckets`. + +--- + +**Issue**: Objects are being skipped +**Solution**: Check the object is under the maximum object size, matches the include filter if one is set, and does not match the exclude filter. diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go index 413dde690dae..571c0ac7049b 100644 --- a/pkg/sources/gcs/gcs.go +++ b/pkg/sources/gcs/gcs.go @@ -99,7 +99,7 @@ func newPersistableCache(increment int, cache cache.Cache[string], p *sources.Pr func (c *persistableCache) Set(key string, val string) { c.Cache.Set(key, val) if ok, contents := c.shouldPersist(); ok { - c.Progress.EncodedResumeInfo = contents + c.EncodedResumeInfo = contents } } @@ -261,7 +261,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . return fmt.Errorf("error listing objects: %w", err) } s.chunksCh = chunksChan - s.Progress.Message = "starting to process objects..." + s.Message = "starting to process objects..." var wg sync.WaitGroup for obj := range objectCh { @@ -295,8 +295,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . func (s *Source) setupCache(ctx context.Context) *persistableCache { var c cache.Cache[string] - if s.Progress.EncodedResumeInfo != "" { - keys := strings.Split(s.Progress.EncodedResumeInfo, ",") + if s.EncodedResumeInfo != "" { + keys := strings.Split(s.EncodedResumeInfo, ",") entries := make([]simple.CacheEntry[string], len(keys)) for i, val := range keys { entries[i] = simple.CacheEntry[string]{Key: val, Value: val} @@ -321,14 +321,14 @@ func (s *Source) setProgress(ctx context.Context, md5, objName string, cache cac s.SectionsCompleted++ cache.Set(md5, md5) - s.Progress.SectionsRemaining = int32(s.stats.numObjects) - s.Progress.PercentComplete = int64(float64(s.SectionsCompleted) / float64(s.stats.numObjects) * 100) + s.SectionsRemaining = int32(s.stats.numObjects) + s.PercentComplete = int64(float64(s.SectionsCompleted) / float64(s.stats.numObjects) * 100) } func (s *Source) completeProgress(ctx context.Context) { msg := fmt.Sprintf("GCS source finished processing %d objects", s.stats.numObjects) ctx.Logger().Info(msg) - s.Progress.Message = msg + s.Message = msg } func (s *Source) processObject(ctx context.Context, o object) error { diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.go index 9cde72ffe444..7d3dc65cb801 100644 --- a/pkg/sources/gcs/gcs_test.go +++ b/pkg/sources/gcs/gcs_test.go @@ -400,14 +400,14 @@ func TestSourceChunks_ProgressSet(t *testing.T) { // Test that the resume progress is set. var progress strings.Builder for i := range got { - progress.WriteString(fmt.Sprintf("md5hash%d", i)) + fmt.Fprintf(&progress, "md5hash%d", i) // Add a comma if not the last element. if i != len(got)-1 { progress.WriteString(",") } } - encodeResume := strings.Split(source.Progress.EncodedResumeInfo, ",") + encodeResume := strings.Split(source.EncodedResumeInfo, ",") sort.Slice(encodeResume, func(i, j int) bool { numI, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[i], "md5hash")) numJ, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[j], "md5hash")) @@ -415,9 +415,9 @@ func TestSourceChunks_ProgressSet(t *testing.T) { }) assert.Equal(t, progress.String(), strings.Join(encodeResume, ",")) - assert.Equal(t, int32(defaultCachePersistIncrement), source.Progress.SectionsCompleted) - assert.Equal(t, int64(100), source.Progress.PercentComplete) - assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", defaultCachePersistIncrement), source.Progress.Message) + assert.Equal(t, int32(defaultCachePersistIncrement), source.SectionsCompleted) + assert.Equal(t, int64(100), source.PercentComplete) + assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", defaultCachePersistIncrement), source.Message) } func TestSource_CachePersistence(t *testing.T) { @@ -457,8 +457,8 @@ func TestSource_CachePersistence(t *testing.T) { // Test that the resume progress is empty. // The cache should not have been persisted. - assert.Equal(t, "", source.Progress.EncodedResumeInfo) - assert.Equal(t, int32(wantObjCnt), source.Progress.SectionsCompleted) - assert.Equal(t, int64(100), source.Progress.PercentComplete) - assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", wantObjCnt), source.Progress.Message) + assert.Equal(t, "", source.EncodedResumeInfo) + assert.Equal(t, int32(wantObjCnt), source.SectionsCompleted) + assert.Equal(t, int64(100), source.PercentComplete) + assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", wantObjCnt), source.Message) } diff --git a/pkg/sources/git/README.md b/pkg/sources/git/README.md new file mode 100644 index 000000000000..aecd3e6db4fb --- /dev/null +++ b/pkg/sources/git/README.md @@ -0,0 +1,350 @@ +# Git Source + +## Overview + +The Git source lets TruffleHog scan Git repositories for secrets, credentials, and sensitive data. It reads the full commit history of a repository, not just the files as they look today, so secrets that were committed once and later removed are still found. + +## Git Fundamentals + +### What does this source scan? + +A Git repository keeps every version of every file it has ever tracked. A secret that was committed and then deleted in a later commit still lives in the history. This source walks that history commit by commit and scans what changed in each one. + +### Key Git Terminology + +| Term | Description | +|------|-------------| +| **Commit** | A saved snapshot of changes, identified by a 40 character hash | +| **Diff** | The lines that changed in a file between one commit and the one before it | +| **Branch** | A named pointer to a commit, such as `main` | +| **Ref** | Any named pointer to a commit, which covers branches, tags, and remote tracking names | +| **Clone** | A local copy of a remote repository | +| **Bare repository** | A repository with no working copy of the files, holding only the Git data itself | +| **Mirror clone** | A bare clone that copies every ref from the remote, not only the default branch | +| **Staged changes** | Changes added with `git add` but not committed yet | +| **Merge base** | The commit where two branches last shared history | + +## Features + +- **Full History Scanning**: Walks every commit reachable from every ref, so deleted secrets are still found +- **Commit Metadata Scanning**: Scans the author email, committer, and commit message, not only file changes +- **Multiple Sources**: Scan a remote URL over HTTPS or SSH, or a repository already on disk +- **Multiple Authentication Methods**: Unauthenticated, username and password or token, or SSH +- **Scan Range Control**: Limit the scan to one branch, to commits after a given commit, or to a maximum number of commits +- **Path Filtering**: Include or exclude files by regex, or exclude them by glob at the `git log` level +- **Staged Change Scanning**: Scans changes that are staged but not committed yet, which makes pre commit hook use possible +- **Binary File Handling**: Reads binary files in full through `git cat-file` instead of reading the diff, or skips them +- **Clone Retries**: Retries a failed clone when the failure looks like a network problem or a rate limit + +## Requirements + +The `git` command must be installed and on your `PATH`. The version must be 2.20.0 or newer, and below 3.0.0. TruffleHog checks this when the source starts and fails with a clear message if it is not met. + +## Configuration + +### Repository Location + +The CLI takes the repository as a positional argument, not a flag. The URL must have a scheme, since a plain path like `/home/user/repo` is rejected as an unsupported URI: + +```bash +# HTTPS (http is accepted the same way) +trufflehog git https://github.com/trufflesecurity/test_keys.git + +# SSH +trufflehog git ssh://git@github.com/trufflesecurity/test_keys.git + +# A repository already on disk +trufflehog git file:///path/to/local/repo +``` + +A remote repository is cloned first, then scanned. A `file://` path is also cloned into a separate directory, so the original copy is never written to. + +In the YAML config, use `repositories` for remote URLs to clone, and `directories` for repositories already on disk, which are scanned where they are. + +### Authentication Methods + +#### 1. Unauthenticated + +For public repositories. + +**CLI Usage:** +```bash +trufflehog git https://github.com/trufflesecurity/test_keys.git +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Git + unauthenticated: {} + repositories: + - https://github.com/trufflesecurity/test_keys.git + name: git-scan + type: SOURCE_TYPE_GIT + verify: true +``` + +--- + +#### 2. Basic Authentication + +For private repositories that need a username and a password or token. + +**CLI Usage:** + +There is no separate flag for this. Put the credentials in the URL: + +```bash +trufflehog git https://myuser:mytoken@github.com/myorg/private-repo.git +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Git + basic_auth: + username: myuser + password: mytoken + repositories: + - https://github.com/myorg/private-repo.git + name: git-scan + type: SOURCE_TYPE_GIT + verify: true +``` + +--- + +#### 3. SSH + +Uses the SSH keys already set up on the machine. There is no key or passphrase field to fill in. + +**CLI Usage:** +```bash +trufflehog git ssh://git@github.com/myorg/private-repo.git +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Git + ssh_auth: {} + repositories: + - ssh://git@github.com/myorg/private-repo.git + name: git-scan + type: SOURCE_TYPE_GIT + verify: true +``` + +### Limiting the Scan Range + +**Scanning One Branch** + +By default every ref in the repository is scanned. Pass a branch name to scan only that branch. + +```bash +trufflehog git https://github.com/myorg/myrepo.git --branch main +``` + +**Scanning Since a Commit** + +Scan only the commits made after the given commit. If the repository is on `github.com`, TruffleHog looks up the date of that commit through the GitHub API and does a shallow clone from that date, which makes the clone much smaller. For any other host it falls back to a normal clone and stops walking once it reaches that commit. + +```bash +trufflehog git https://github.com/myorg/myrepo.git --since-commit a1b2c3d4 +``` + +If a `GITHUB_TOKEN` environment variable is set, it is used for that commit lookup, which is needed for private repositories. + +**Limiting Commit Depth** + +Stop after this many commits. + +```bash +trufflehog git https://github.com/myorg/myrepo.git --max-depth 100 +``` + +### Filtering What Gets Scanned + +**Include or Exclude Paths** + +Both flags take a path to a file that holds one regex per line. Blank lines and lines starting with `#` are ignored, so the file can hold comments. + +```bash +trufflehog git https://github.com/myorg/myrepo.git --include-paths ./include.txt +``` + +```bash +trufflehog git https://github.com/myorg/myrepo.git --exclude-paths ./exclude.txt +``` + +**Exclude Globs** + +Takes a comma separated list of globs. This filter is applied at the `git log` level, so the excluded files are never read at all, which makes the scan faster than filtering afterwards. + +```bash +trufflehog git https://github.com/myorg/myrepo.git --exclude-globs "*.min.js,vendor/*" +``` + +### Clone Location and Cleanup + +By default a repository is cloned into a temporary directory and that directory is deleted after the scan. Use `--clone-path` to clone somewhere else, and `--no-cleanup` to keep the clone afterwards. + +```bash +trufflehog git https://github.com/myorg/myrepo.git --clone-path /tmp/my-clones --no-cleanup +``` + +`--no-cleanup` only works together with `--clone-path`, and the path given to `--clone-path` must already exist and be a directory. Each clone gets its own directory inside it, so disk use grows with every repository and every run. + +Warning: cleanup deletes the directory that was scanned, and it is keyed on `--clone-path` being set rather than on whether that directory was actually cloned by TruffleHog. So when `--clone-path` is set and `--no-cleanup` is not, a repository that was scanned in place gets deleted too. That applies to `--trust-local-git-config` on the CLI, and to `directories` entries in the YAML config. Do not combine either of those with `--clone-path` or `clone_path`. + +### Other Options + +**Bare Repository** + +Scan a repository that has no working copy, which is useful in a pre receive hook. + +```bash +trufflehog git file:///path/to/repo.git --bare +``` + +**Trust Local Git Config** + +For a `file://` path, scan the repository where it already is instead of cloning it first. This makes TruffleHog read the local Git config of that repository. + +```bash +trufflehog git file:///path/to/local/repo --trust-local-git-config +``` + +Do not pass `--clone-path` alongside this flag, for the reason given in the cleanup warning above. + +## How Scanning Works + +### Scanning Process + +1. **Git Check**: Confirms the `git` command is installed and its version is supported. +2. **Clone or Open**: A remote URL is cloned into a temporary directory or into `--clone-path`. A repository already on disk is opened where it is. +3. **Commit Walk**: Runs `git log` over the repository with full history, across every ref by default, or over one branch when `--branch` is given. +4. **Commit Metadata Chunk**: For each commit, the author email, the committer, and the commit message are sent to the detection engine as their own chunk. +5. **File Diff Chunks**: For each changed file in the commit, the diff is sent as a chunk tagged with the commit hash, file name, author email, timestamp, repository, and line number. A diff larger than the chunk size is split into several chunks line by line. +6. **Binary Files**: A binary file is not read from the diff. Its full contents are pulled with `git cat-file` and passed through the file handlers, which also unpack archives. +7. **Staged Changes**: If the repository is not bare, `git diff --cached` is scanned as well, so changes staged but not committed are covered. +8. **Cleanup**: The clone is deleted unless `--no-cleanup` was used with `--clone-path`. + +### Clone Retries + +A clone that fails because of a network problem or what looks like a rate limit is retried, up to 3 attempts in total, each from a fresh directory. Network failures wait 5 seconds times the attempt number, and rate limit failures wait 60 seconds times the attempt number, since those take longer to clear. Any other failure, such as a bad password or a missing repository, is returned right away without retrying. + +### What Gets Scanned + +- The diff of every added or modified file in every commit. When `--since-commit` is used, deletions and renames are included as well +- Commit metadata: author email, committer, and commit message +- Binary files, read in full rather than as a diff +- Files inside archives found in the repository +- Staged changes, when the repository is not bare + +### What Doesn't Get Scanned + +- Files excluded by the include paths, exclude paths, or exclude globs filters +- Commits past `--max-depth`, or reachable from the commit given to `--since-commit` (that commit and its ancestors) +- Refs other than the one named by `--branch`, when that flag is used +- Binary files, when `--force-skip-binaries` is used +- Binary files whose extension TruffleHog already skips, such as common image, audio, video, and font types +- Files inside archives, when `--force-skip-archives` is used +- Staged changes in a bare repository, since a bare repository has nothing staged + +## Usage Examples + +### Scanning a Public Repository + +```bash +trufflehog git https://github.com/trufflesecurity/test_keys.git +``` + +### Scanning a Local Repository + +```bash +trufflehog git file:///path/to/local/repo +``` + +### Scanning One Branch Only + +```bash +trufflehog git https://github.com/myorg/myrepo.git --branch main +``` + +### Scanning Only Recent History + +```bash +trufflehog git https://github.com/myorg/myrepo.git --max-depth 50 +``` + +### Scanning a Private Repository Over SSH + +```bash +trufflehog git ssh://git@github.com/myorg/private-repo.git +``` + +### Keeping the Clone After the Scan + +```bash +trufflehog git https://github.com/myorg/myrepo.git --clone-path /tmp/my-clones --no-cleanup +``` + +## Pre Commit Hook Use + +TruffleHog notices when it is being run as a pre commit hook and changes some settings on its own: + +- Local Git config is trusted +- Only staged changes are scanned +- Only verified and unknown results are shown +- The run fails if anything is found, which stops the commit + +It detects this from these environment variables: + +| Variable | Set by | +|----------|--------| +| `PRE_COMMIT=1` | The pre-commit framework | +| `HUSKY=1` | Husky, modern versions | +| `HUSKY_GIT_PARAMS` | Husky, versions below 4.0 | +| `TRUFFLEHOG_PRE_COMMIT=1` | Set by hand in a plain Git hook script | + +For a plain Git hook with no framework, export the variable yourself in `.git/hooks/pre-commit`: + +```bash +export TRUFFLEHOG_PRE_COMMIT=1 +``` + +## Troubleshooting + +### Common Issues + +**Issue**: `'git' command not found in $PATH` +**Solution**: Install Git and make sure it is on your `PATH`. The version must be 2.20.0 or newer and below 3.0.0. + +--- + +**Issue**: Authentication failures when cloning a private repository +**Solution**: For HTTPS, check the username and token in the URL or in the config, and that the token can read the repository. For SSH, check that the key on the machine is loaded and accepted by the host. + +--- + +**Issue**: `--no-cleanup can only be used together with --clone-path` +**Solution**: `--no-cleanup` keeps the clone in place, so a path to keep it in is needed. Pass `--clone-path` as well, pointing at a directory that already exists. + +--- + +**Issue**: Running out of disk space during a scan +**Solution**: Every repository is cloned in full, so a large history needs a lot of space. Drop `--no-cleanup` so clones are deleted after each scan, or use `--since-commit` on a `github.com` repository so the clone is shallow. + +--- + +**Issue**: The scan is slow on a large repository +**Solution**: Narrow it with `--branch`, `--max-depth`, or `--since-commit`. Use `--exclude-globs` rather than `--exclude-paths`, since globs are filtered inside `git log` and those files are never read. + +--- + +**Issue**: Clones keep failing on a host that rate limits +**Solution**: TruffleHog already retries a rate limited clone 3 times, waiting longer between each try. If it still fails, wait and scan fewer repositories at once. diff --git a/pkg/sources/git/cmd_check.go b/pkg/sources/git/cmd_check.go index 67df435636d0..be8062af6eb8 100644 --- a/pkg/sources/git/cmd_check.go +++ b/pkg/sources/git/cmd_check.go @@ -2,8 +2,8 @@ package git import ( "fmt" + regexp "github.com/wasilibs/go-re2" "os/exec" - "regexp" "strconv" "strings" diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 57f3498961f6..326188dcc844 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -6,12 +6,12 @@ import ( "encoding/base64" "errors" "fmt" + regexp "github.com/wasilibs/go-re2" "io" "net/url" "os" "os/exec" "path/filepath" - "regexp" "runtime" "strings" "sync/atomic" @@ -38,7 +38,22 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) -const SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT +const ( + SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT + // maxCloneAttempts is the total number of times a clone is attempted when + // each failure is classified as a transient network or rate-limit error. + maxCloneAttempts = 3 + // cloneRetryBackoff is the base wait between clone attempts after a + // transient network error; it is multiplied by the number of failed + // attempts so far. + cloneRetryBackoff = 5 * time.Second + // cloneRateLimitBackoff is the base wait after a clone attempt fails + // with what looks like a GitHub/GitLab secondary rate limit (bare 403 + // or 429). GitHub's guidance for secondary rate limits is to wait at + // least a minute before retrying, so this starts well above the + // network-error backoff and grows per attempt. + cloneRateLimitBackoff = 60 * time.Second +) type Source struct { name string @@ -112,11 +127,12 @@ type Config struct { // NewGit creates a new Git instance with the provided configuration. The Git instance is used to interact with // Git repositories. func NewGit(config *Config) *Git { - var parser *gitparse.Parser + parserOpts := []gitparse.Option{} if config.UseCustomContentWriter { - parser = gitparse.NewParser(gitparse.UseCustomContentWriter()) - } else { - parser = gitparse.NewParser() + parserOpts = append(parserOpts, gitparse.UseCustomContentWriter()) + } + if feature.UseGitLowMemoryScan.Load() { + parserOpts = append(parserOpts, gitparse.UseLowMemoryScan()) } return &Git{ @@ -130,7 +146,7 @@ func NewGit(config *Config) *Git { concurrency: semaphore.NewWeighted(int64(config.Concurrency)), skipBinaries: config.SkipBinaries, skipArchives: config.SkipArchives, - parser: parser, + parser: gitparse.NewParser(parserOpts...), } } @@ -319,7 +335,7 @@ func (s *Source) scanRepo(ctx context.Context, repoURI string, reporter sources. // if legacy JSON is enabled, don't remove the directory because we need it for outputting legacy JSON. if !s.conn.GetPrintLegacyJson() { if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.conn.GetNoCleanup() && s.conn.GetClonePath() != "") { - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() } } @@ -373,7 +389,7 @@ func (s *Source) scanDir(ctx context.Context, gitDir string, reporter sources.Ch // if legacy JSON is enabled, don't remove the directory because we need it for outputting legacy JSON. if !s.conn.GetPrintLegacyJson() { if strings.HasPrefix(gitDir, filepath.Join(os.TempDir(), "trufflehog")) || (!s.conn.GetNoCleanup() && s.conn.GetClonePath() != "") { - defer os.RemoveAll(gitDir) + defer func() { _ = os.RemoveAll(gitDir) }() } } @@ -404,7 +420,7 @@ func RepoFromPath(path string) (*git.Repository, error) { func CleanOnError(err *error, path string) { if *err != nil { - os.RemoveAll(path) + _ = os.RemoveAll(path) } } @@ -474,38 +490,127 @@ type cloneParams struct { // infrastructure, ensuring that any encountered errors trigger a cleanup of resources. // The core cloning logic is delegated to a nested function, which returns errors to the // outer function for centralized error handling and cleanup. +// +// Failures classified as transient network errors (e.g. a connection reset +// mid-transfer) or as a secondary rate limit (a bare 403 or 429 from the +// remote) are retried up to maxCloneAttempts times, each attempt starting +// from a fresh clone directory. All other failures, including clone +// timeouts (see feature.GitCloneTimeoutDuration), are returned immediately. func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitURL string, clonePath string, authInUrl bool, args ...string) (string, *git.Repository, error) { + timeout := time.Duration(feature.GitCloneTimeoutDuration.Load()) + var path string + var repo *git.Repository var err error + for attempt := 1; attempt <= maxCloneAttempts; attempt++ { + // Each attempt starts from a fresh clone directory. + path, err = createClonePath(gitURL, clonePath) + if err != nil { + return "", nil, err + } - // If --clone-path is set, create a subdirectory /trufflehog- with permissions 0755. - if clonePath != "" { - path = filepath.Join(clonePath, "trufflehog-"+strings.TrimSuffix(filepath.Base(gitURL), gitDirName)) - if err = os.MkdirAll(path, 0755); err != nil { - return "", nil, fmt.Errorf("failed to create clone path %s: %w", clonePath, err) + repo, err = executeClone(ctx, cloneParams{userInfo, gitURL, args, path, authInUrl, timeout}) + if err == nil { + return path, repo, nil } - } else { - // otherwise, create a temporary directory in the system temp path. - path, err = cleantemp.MkdirTemp() + + if attempt >= maxCloneAttempts || !isRetryableCloneError(err) || common.IsDone(ctx) { + // DO NOT FORGET TO CLEAN UP THE CLONE PATH HERE!! + // If we don't, we'll end up with a bunch of orphaned directories in the temp dir. + CleanOnError(&err, path) + + // Note: We don't need to record the clone failure here as it's already + // recorded in executeClone when the error occurs + return "", nil, err + } + + // Discard the partial clone; the next iteration recreates a fresh directory. + if rmErr := os.RemoveAll(path); rmErr != nil { + return "", nil, fmt.Errorf("failed to clean clone path for retry: %w (original clone error: %w)", rmErr, err) + } + + delay := cloneRetryDelay(err, attempt) + ctx.Logger().Info("git clone interrupted; retrying", + "attempt", attempt, + "max_attempts", maxCloneAttempts, + "delay", delay.String(), + "error", err.Error(), + ) + select { + case <-ctx.Done(): + return "", nil, ctx.Err() + case <-time.After(delay): + } + } + + return "", nil, err +} + +// isRetryableCloneError reports whether a clone failure looks like a +// transient condition (a network interruption, e.g. a connection reset +// mid-transfer, or a secondary rate limit) rather than a permanent +// condition like an auth or permission error. +func isRetryableCloneError(err error) bool { + if err == nil { + return false + } + switch ClassifyCloneError(err.Error()) { + case cloneFailureNetwork, cloneFailureRateLimit: + return true + default: + return false + } +} + +// cloneRetryDelay returns how long to wait before the next clone attempt, +// scaled by the failure class: rate-limit errors back off much more slowly +// than transient network errors, since GitHub/GitLab secondary rate limits +// take on the order of a minute or more to clear. +func cloneRetryDelay(err error, attempt int) time.Duration { + if ClassifyCloneError(err.Error()) == cloneFailureRateLimit { + return cloneRateLimitBackoff * time.Duration(attempt) + } + return cloneRetryBackoff * time.Duration(attempt) +} + +// createClonePath creates the directory a repository will be cloned into and +// returns its path. When clonePath is set (the --clone-path flag), the +// directory is /trufflehog-- with permissions +// 0755; otherwise a fresh temporary directory (0700, per os.MkdirTemp) is +// created in the system temp path. It is called both before the first clone +// attempt and to replace the directory between retries. +// +// Every call returns a directory of its own. Naming it after the URL's last +// segment alone collides for same-named repos in different groups, and for +// the same repo cloned twice, which lets concurrent workers clone into and +// delete each other's directories. +func createClonePath(gitURL, clonePath string) (string, error) { + if clonePath == "" { + path, err := cleantemp.MkdirTemp() if err != nil { - return "", nil, fmt.Errorf("failed to create temporary clone path: %w", err) + return "", fmt.Errorf("failed to create temporary clone path: %w", err) } + return path, nil } - timeout := time.Duration(feature.GitCloneTimeoutDuration.Load()) + if err := os.MkdirAll(clonePath, 0755); err != nil { + return "", fmt.Errorf("failed to create clone path %s: %w", clonePath, err) + } - repo, err := executeClone(ctx, cloneParams{userInfo, gitURL, args, path, authInUrl, timeout}) + // The trufflehog- prefix is what cleantemp.CleanTempDirsForLegacyJSON + // sweeps, so it has to survive; the repo name is kept for readability + // under --no-cleanup. + slug := strings.TrimSuffix(filepath.Base(gitURL), gitDirName) + path, err := os.MkdirTemp(clonePath, "trufflehog-"+slug+"-") if err != nil { - // DO NOT FORGET TO CLEAN UP THE CLONE PATH HERE!! - // If we don't, we'll end up with a bunch of orphaned directories in the temp dir. - CleanOnError(&err, path) - - // Note: We don't need to record the clone failure here as it's already - // recorded in executeClone when the error occurs - return "", nil, err + return "", fmt.Errorf("failed to create clone path in %s: %w", clonePath, err) } - return path, repo, nil + // os.MkdirTemp creates 0700; --clone-path directories are 0755. + if err := os.Chmod(path, 0755); err != nil { + return "", fmt.Errorf("failed to set permissions on clone path %s: %w", path, err) + } + return path, nil } // executeClone prepares the Git URL, constructs, and executes the git clone command using the provided @@ -591,7 +696,7 @@ func executeClone(ctx context.Context, params cloneParams) (*git.Repository, err } else if cloneCmd.ProcessState == nil { return nil, fmt.Errorf("clone command exited with no output") } else if cloneCmd.ProcessState.ExitCode() != 0 { - logger.V(1).Info("git clone failed", "error", err) + logger.V(0).Info("git clone failed", "error", err) failureReason := ClassifyCloneError(output) exitCode := cloneCmd.ProcessState.ExitCode() metricsInstance.RecordCloneOperation(statusFailure, failureReason, exitCode) @@ -678,6 +783,50 @@ func (s *Git) CommitsScanned() uint64 { const gitDirName = ".git" +// resolveGitDir resolves the actual git directory path for a repository. +// In a regular repository, .git is a directory containing the git data. +// In a git worktree, .git is a file containing a "gitdir: " reference +// to the actual git directory location. +// This function handles both cases and returns the path to the actual git directory. +func resolveGitDir(repoPath string) (string, error) { + gitPath := filepath.Join(repoPath, gitDirName) + + info, err := os.Stat(gitPath) + if err != nil { + return "", fmt.Errorf("failed to stat .git: %w", err) + } + + // If .git is a directory, return it directly + if info.IsDir() { + return gitPath, nil + } + + // .git is a file (worktree) - read and parse the gitdir reference + content, err := os.ReadFile(gitPath) + if err != nil { + return "", fmt.Errorf("failed to read .git file: %w", err) + } + + // Parse "gitdir: " format + line := strings.TrimSpace(string(content)) + const gitdirPrefix = "gitdir: " + if !strings.HasPrefix(line, gitdirPrefix) { + return "", fmt.Errorf("invalid .git file format: expected 'gitdir: ', got %q", line) + } + + gitdirPath := strings.TrimPrefix(line, gitdirPrefix) + + // The path may be relative to the worktree directory + if !filepath.IsAbs(gitdirPath) { + gitdirPath = filepath.Join(repoPath, gitdirPath) + } + + // Clean the path to resolve any ".." components + gitdirPath = filepath.Clean(gitdirPath) + + return gitdirPath, nil +} + // getGitDir returns the likely path of the ".git" directory. // If the repository is bare, it will be at the top-level; otherwise, it // exists in the ".git" directory at the root of the working tree. @@ -707,6 +856,13 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string repoCtx = ctx } + // The scan can stop before the diff channel is drained, on max depth or on + // reaching the base commit. Nothing else tells the parser that, so cancelling on + // the way out is what shuts down the git processes still producing diffs. Without + // it they sit blocked on a channel nobody is reading until the whole scan ends. + repoCtx, cancel := context.WithCancel(repoCtx) + defer cancel() + logger := repoCtx.Logger() var logValues []any if scanOptions.BaseHash != "" { @@ -719,7 +875,8 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string logValues = append(logValues, "max_depth", scanOptions.MaxDepth) } - diffChan, err := s.parser.RepoPath(repoCtx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, isRepoBare(path)) + // git computes the base..head range, so every commit on diffChan is in scope. + diffChan, err := s.parser.RepoPath(repoCtx, path, scanOptions.HeadHash, scanOptions.BaseHash, scanOptions.ExcludeGlobs, isRepoBare(path)) if err != nil { return err } @@ -743,14 +900,9 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string commit := diff.Commit fullHash := commit.Hash - if scanOptions.BaseHash != "" && scanOptions.BaseHash == fullHash { - logger.V(1).Info("reached base commit", "commit", fullHash) - break - } email := commit.Author when := commit.Date.UTC().Format("2006-01-02 15:04:05 -0700") - if fullHash != lastCommitHash { depth++ lastCommitHash = fullHash @@ -863,7 +1015,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string ) return nil } - defer reader.Close() + defer func() { _ = reader.Close() }() data := make([]byte, d.Len()) if _, err := io.ReadFull(reader, data); err != nil { @@ -889,6 +1041,11 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string return err } } + + // empty base..head range exits successfully + if scanOptions.BaseHash != "" && depth == 0 { + logger.Info("no commits in range", logValues...) + } return nil } @@ -898,9 +1055,17 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email ctx.Logger().Error(err, "error creating reader for chunk", "filename", fileName, "commit", hash, "file", diff.PathB) return } - defer reader.Close() + defer func() { _ = reader.Close() }() originalChunk := bufio.NewScanner(reader) + // Default bufio max token size (64 KB) is too small for files with long lines + // (e.g. minified JS, base64 blobs). Raise the cap to 10 MB so those lines + // are still scanned; the oversize-line path below will chunk them correctly. + // The initial buffer starts at 4 KB (same as bufio's default) and grows only + // when a line actually exceeds the current size, keeping allocations cheap for + // the common case of small diffs. + const maxScanTokenSize = 10 * 1024 * 1024 + originalChunk.Buffer(make([]byte, 4096), maxScanTokenSize) newChunkBuffer := bytes.Buffer{} lastOffset := 0 for offset := 0; originalChunk.Scan(); offset++ { @@ -967,6 +1132,20 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email ctx.Logger().Error(err, "error writing to chunk buffer", "filename", fileName, "commit", hash, "file", diff.PathB) } } + if err := originalChunk.Err(); err != nil { + // A single diff line exceeding maxScanTokenSize (e.g. minified JS or a + // base64 blob) is an expected condition for pathological files, not a + // failure. bufio.Scanner stops after this line, so the remainder of the + // diff is not scanned; log at a lower level to keep it out of the error + // stream while still recording that content was truncated. Genuine reader + // errors are still surfaced at ERROR. + if errors.Is(err, bufio.ErrTooLong) { + ctx.Logger().V(2).Info("skipping oversize diff line; remainder of file not scanned", + "filename", fileName, "commit", hash, "file", diff.PathB, "max_line_bytes", maxScanTokenSize) + } else { + ctx.Logger().Error(err, "error scanning chunk", "filename", fileName, "commit", hash, "file", diff.PathB) + } + } // Send anything still in the new chunk buffer if newChunkBuffer.Len() > 0 { metadata := s.sourceMetadataFunc(SourceMetadataInfo{ @@ -1115,7 +1294,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string, logger.Error(err, "error creating reader for staged") return nil } - defer reader.Close() + defer func() { _ = reader.Close() }() data := make([]byte, d.Len()) if _, err := reader.Read(data); err != nil { @@ -1418,8 +1597,13 @@ func PrepareRepo(ctx context.Context, uriString, clonePath string, trustLocalGit // Note: To scan **un**staged changes in the future, we'd need to set core.worktree to the original path. uriPath := normalizedURI.Path - originalIndexPath := filepath.Join(uriPath, gitDirName, "index") + // Resolve the actual git directory (handles both regular repos and worktrees) + originalGitDir, err := resolveGitDir(uriPath) + if err != nil { + return path, remote, fmt.Errorf("failed to resolve git directory: %w", err) + } + originalIndexPath := filepath.Join(originalGitDir, "index") clonedIndexPath := filepath.Join(path, gitDirName, "index") indexData, err := os.ReadFile(originalIndexPath) @@ -1429,6 +1613,24 @@ func PrepareRepo(ctx context.Context, uriString, clonePath string, trustLocalGit if err := os.WriteFile(clonedIndexPath, indexData, 0644); err != nil { return path, remote, fmt.Errorf("failed to write index file: %w", err) } + + // Add the source object store as an alternate so staged blobs are accessible. + // git clone with file:// only transfers reachable objects; staged blobs must + // be reached via the original object store. + // For worktrees, the commondir file points to the main repo's git dir + // which holds the actual shared object store. + sourceObjectsPath := filepath.Join(originalGitDir, "objects") + if commondirData, err := os.ReadFile(filepath.Join(originalGitDir, "commondir")); err == nil { + commondir := strings.TrimSpace(string(commondirData)) + if !filepath.IsAbs(commondir) { + commondir = filepath.Join(originalGitDir, commondir) + } + sourceObjectsPath = filepath.Join(filepath.Clean(commondir), "objects") + } + alternatesPath := filepath.Join(path, gitDirName, "objects", "info", "alternates") + if err := os.MkdirAll(filepath.Dir(alternatesPath), 0755); err == nil { + _ = os.WriteFile(alternatesPath, []byte(sourceObjectsPath+"\n"), 0644) + } } } case "http", "https": diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go index f419eee5e0fd..07db93a2a856 100644 --- a/pkg/sources/git/git_test.go +++ b/pkg/sources/git/git_test.go @@ -2,12 +2,14 @@ package git import ( "bytes" + "errors" "fmt" "os" "os/exec" "path/filepath" "runtime" "strings" + "sync" "testing" "time" @@ -60,6 +62,323 @@ func TestClone_Timeout(t *testing.T) { }) } +func TestIsRetryableCloneError(t *testing.T) { + retryable := []string{ + // Connection reset mid-transfer. + `could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, error: RPC failed; curl 56 Recv failure: Connection reset by peer +error: 7589 bytes of body are still expected +fetch-pack: unexpected disconnect while reading sideband packet +fatal: early EOF +fatal: fetch-pack: invalid index-pack output`, + // HTTP/2 stream reset by the server. + `could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, error: RPC failed; curl 92 HTTP/2 stream 7 reset by server (error 0x8 CANCEL) +error: 7457 bytes of body are still expected +fetch-pack: unexpected disconnect while reading sideband packet +fatal: early EOF +fatal: fetch-pack: invalid index-pack output`, + // Bare 403/429 during clone matches GitHub/GitLab secondary rate + // limiting (no accompanying auth/permission message). + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, fatal: unable to access 'https://github.com/org/repo.git/': The requested URL returned error: 403", + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, The requested URL returned error: 429", + // Normal clone progress ("remote: Counting objects...") must not be + // mistaken for a denial explanation when a later 403 is just throttling. + `could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, remote: Enumerating objects: 100, done. +remote: Counting objects: 100% (100/100), done. +fatal: unable to access 'https://github.com/org/repo.git/': The requested URL returned error: 403`, + } + for _, msg := range retryable { + assert.True(t, isRetryableCloneError(errors.New(msg)), "expected retryable: %q", msg) + } + + notRetryable := []string{ + // An explicit permission-denial message, as opposed to a bare 403. + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, remote: You are not allowed to download code from this project.", + // The explicit denial message must win even when the combined clone + // output also contains a literal "403" elsewhere. + "could not clone repo: https://gitlab.com/org/repo.git, error executing git clone: exit status 128, fatal: unable to access 'https://gitlab.com/org/repo.git/': The requested URL returned error: 403\nremote: You are not allowed to download code from this project.", + // GitHub's explicit permission-denial message, which also co-occurs + // with a literal "403" in the combined clone output. + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, remote: Permission to org/repo.git denied to user.\nfatal: unable to access 'https://github.com/org/repo.git/': The requested URL returned error: 403", + // GitHub Apps/fine-grained-token variant: "Write access ... not + // granted", also co-occurring with a literal 403. + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, remote: Write access to repository not granted.\nfatal: unable to access 'https://github.com/org/repo.git/': The requested URL returned error: 403", + // SAML SSO enforcement: an unrecognized "remote:" explanation + // co-occurring with a 403, which the classifier must treat as a + // permanent failure without needing this exact wording enumerated. + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, remote: The organization has enabled or enforced SAML SSO.\nfatal: unable to access 'https://github.com/org/repo.git/': The requested URL returned error: 403", + "git clone timed out (after 1h0m0s)", + "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, fatal: repository 'https://github.com/org/repo.git/' not found", + } + for _, msg := range notRetryable { + assert.False(t, isRetryableCloneError(errors.New(msg)), "expected not retryable: %q", msg) + } + assert.False(t, isRetryableCloneError(nil)) +} + +func TestCloneRetryDelay(t *testing.T) { + rateLimitErr := errors.New("The requested URL returned error: 429") + networkErr := errors.New("fatal: early EOF") + + // Rate limits take longer to clear, so they back off from a much larger + // base than transient network errors. + assert.Equal(t, cloneRateLimitBackoff, cloneRetryDelay(rateLimitErr, 1)) + assert.Equal(t, 2*cloneRateLimitBackoff, cloneRetryDelay(rateLimitErr, 2)) + + assert.Equal(t, cloneRetryBackoff, cloneRetryDelay(networkErr, 1)) + assert.Equal(t, 2*cloneRetryBackoff, cloneRetryDelay(networkErr, 2)) +} + +func TestStripPassword(t *testing.T) { + tests := []struct { + name string + url string + wantURL string + wantPassword string + }{ + { + name: "username and password are removed", + url: "https://user:pass@github.com/org/repo.git", + wantURL: "https://github.com/org/repo.git", + wantPassword: "pass", + }, + { + name: "username without a password is removed", + url: "https://user@github.com/org/repo.git", + wantURL: "https://github.com/org/repo.git", + wantPassword: "", + }, + { + name: "url without credentials is unchanged", + url: "https://github.com/org/repo.git", + wantURL: "https://github.com/org/repo.git", + wantPassword: "", + }, + { + name: "scp style git@ url is returned as is", + url: "git@github.com:org/repo.git", + wantURL: "git@github.com:org/repo.git", + wantPassword: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotURL, gotPassword, err := stripPassword(tt.url) + assert.NoError(t, err) + assert.Equal(t, tt.wantURL, gotURL) + assert.Equal(t, tt.wantPassword, gotPassword) + }) + } +} + +func TestCreateClonePath(t *testing.T) { + t.Run("temp dir when clonePath is empty", func(t *testing.T) { + path, err := createClonePath("https://github.com/org/repo.git", "") + assert.NoError(t, err) + defer func() { _ = os.RemoveAll(path) }() + + info, err := os.Stat(path) + assert.NoError(t, err) + assert.True(t, info.IsDir()) + if runtime.GOOS != "windows" { + // os.MkdirTemp guarantees 0700 so private repo contents aren't + // exposed to other users on shared systems. + assert.Equal(t, os.FileMode(0700), info.Mode().Perm()) + } + }) + + t.Run("temp dirs are unique across calls", func(t *testing.T) { + // A retry replaces the previous directory; a colliding name would + // mean cloning into a non-empty directory, which git refuses. + first, err := createClonePath("https://github.com/org/repo.git", "") + assert.NoError(t, err) + defer func() { _ = os.RemoveAll(first) }() + + second, err := createClonePath("https://github.com/org/repo.git", "") + assert.NoError(t, err) + defer func() { _ = os.RemoveAll(second) }() + + assert.NotEqual(t, first, second) + }) + + t.Run("clonePath set builds trufflehog-- subdirectory", func(t *testing.T) { + base := t.TempDir() + path, err := createClonePath("https://github.com/org/repo.git", base) + assert.NoError(t, err) + assert.Equal(t, base, filepath.Dir(path)) + // The trufflehog- prefix is what CleanTempDirsForLegacyJSON sweeps. + assert.True(t, strings.HasPrefix(filepath.Base(path), "trufflehog-repo-"), + "unexpected directory name %q", filepath.Base(path)) + + info, err := os.Stat(path) + assert.NoError(t, err) + assert.True(t, info.IsDir()) + if runtime.GOOS != "windows" { + assert.Equal(t, os.FileMode(0755), info.Mode().Perm()) + } + }) + + t.Run("repo name without .git suffix", func(t *testing.T) { + base := t.TempDir() + path, err := createClonePath("https://github.com/org/repo", base) + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filepath.Base(path), "trufflehog-repo-"), + "unexpected directory name %q", filepath.Base(path)) + }) + + t.Run("trailing slash in repo URL", func(t *testing.T) { + base := t.TempDir() + path, err := createClonePath("https://github.com/org/repo.git/", base) + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filepath.Base(path), "trufflehog-repo-"), + "unexpected directory name %q", filepath.Base(path)) + }) + + t.Run("nonexistent clonePath parents are created", func(t *testing.T) { + base := filepath.Join(t.TempDir(), "a", "b", "c") + path, err := createClonePath("https://github.com/org/repo.git", base) + assert.NoError(t, err) + assert.Equal(t, base, filepath.Dir(path)) + + info, err := os.Stat(path) + assert.NoError(t, err) + assert.True(t, info.IsDir()) + }) + + t.Run("second call with same arguments gets a fresh directory", func(t *testing.T) { + // The retry path calls this again after RemoveAll, and concurrent + // workers may be scanning the same repo; neither may be handed a + // directory another caller already owns. + base := t.TempDir() + first, err := createClonePath("https://github.com/org/repo.git", base) + assert.NoError(t, err) + + second, err := createClonePath("https://github.com/org/repo.git", base) + assert.NoError(t, err) + assert.NotEqual(t, first, second) + }) + + t.Run("error when clonePath location is not writable", func(t *testing.T) { + // Create a *file* where the clone path should go so MkdirAll fails. + base := filepath.Join(t.TempDir(), "blocker") + assert.NoError(t, os.WriteFile(base, []byte("x"), 0644)) + + path, err := createClonePath("https://github.com/org/repo.git", base) + assert.Error(t, err) + assert.Empty(t, path) + assert.Contains(t, err.Error(), "failed to create clone path") + }) + + t.Run("distinct repos sharing a basename get distinct paths", func(t *testing.T) { + // Only the last URL segment is used as the directory slug, so repos + // that live under different groups but share a name collide. With + // concurrency > 1 two workers then clone into and delete the same + // directory, producing spurious clone errors and partial scans. + base := t.TempDir() + + urls := []string{ + "https://gitlab.com/group-a/api.git", + "https://gitlab.com/group-b/api.git", + "https://gitlab.com/group-b/subgroup/api.git", + "https://gitlab.example.com/other/api", + } + + seen := make(map[string]string, len(urls)) + for _, u := range urls { + path, err := createClonePath(u, base) + assert.NoError(t, err) + if prev, ok := seen[path]; ok { + t.Errorf("clone path collision: %q and %q both resolve to %q", prev, u, path) + } + seen[path] = u + } + }) + + t.Run("concurrent calls for the same repo get distinct paths", func(t *testing.T) { + // The same repo can be cloned concurrently (e.g. a unit retried while + // another worker still holds the directory). Each caller owns its + // directory, so no two callers may be handed the same one. + base := t.TempDir() + + const workers = 8 + var ( + mu sync.Mutex + paths = make(map[string]int, workers) + wg sync.WaitGroup + start = make(chan struct{}) + ) + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + path, err := createClonePath("https://gitlab.com/group-a/api.git", base) + assert.NoError(t, err) + mu.Lock() + paths[path]++ + mu.Unlock() + }() + } + close(start) + wg.Wait() + + assert.Len(t, paths, workers, "expected %d distinct clone paths, got %v", workers, paths) + }) +} + +// TestCloneRepo_ConcurrentSameBasename reproduces the failure end to end: two +// workers cloning different repos that share a basename into a shared +// --clone-path. They are handed the same directory, so one clone fails +// ("already exists and is not an empty directory") or one worker's cleanup +// deletes the other's working tree mid-scan. +func TestCloneRepo_ConcurrentSameBasename(t *testing.T) { + ctx := context.Background() + clonePath := t.TempDir() + + // Two distinct source repos that both end in "api". + sources := make([]string, 2) + for i, group := range []string{"group-a", "group-b"} { + repoPath := filepath.Join(t.TempDir(), group, "api") + assert.NoError(t, os.MkdirAll(filepath.Dir(repoPath), 0755)) + assert.NoError(t, exec.Command("git", "init", repoPath).Run()) + assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "user.name", "Test User").Run()) + assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "user.email", "test@example.com").Run()) + assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "commit.gpgsign", "false").Run()) + addTestFileAndCommit(t, repoPath, "secret.txt", "content for "+group) + sources[i] = "file://" + repoPath + } + + var ( + wg sync.WaitGroup + start = make(chan struct{}) + mu sync.Mutex + errs []error + dests = make(map[string]int) + ) + for _, gitURL := range sources { + wg.Add(1) + go func() { + defer wg.Done() + <-start + path, _, err := CloneRepo(ctx, nil, gitURL, clonePath, false) + mu.Lock() + defer mu.Unlock() + if err != nil { + errs = append(errs, err) + return + } + dests[path]++ + // Mirrors the cleanup the callers do once a repo is scanned. + _ = os.RemoveAll(path) + }() + } + close(start) + wg.Wait() + + assert.Empty(t, errs, "concurrent clones of same-named repos should not fail") + assert.Len(t, dests, len(sources), "each repo should clone into its own directory, got %v", dests) +} + func TestSource_Scan(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -650,6 +969,7 @@ func setupTestRepo(t *testing.T, repoName string) string { assert.NoError(t, exec.Command("git", "init", repoPath).Run()) assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "user.name", "Test User").Run()) assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "user.email", "test@example.com").Run()) + assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "commit.gpgsign", "false").Run()) return repoPath } @@ -853,6 +1173,111 @@ func TestPrepareRepoErrorPaths(t *testing.T) { }) } +func TestResolveGitDir(t *testing.T) { + t.Parallel() + + t.Run("regular repository with .git directory", func(t *testing.T) { + repoPath := setupTestRepo(t, "regular-repo") + addTestFileAndCommit(t, repoPath, "test.txt", "test content") + + gitDir, err := resolveGitDir(repoPath) + assert.NoError(t, err) + assert.Equal(t, filepath.Join(repoPath, ".git"), gitDir) + + // Verify it's actually a directory + info, err := os.Stat(gitDir) + assert.NoError(t, err) + assert.True(t, info.IsDir()) + }) + + t.Run("git worktree with .git file", func(t *testing.T) { + // Create main repository + mainRepoPath := setupTestRepo(t, "main-repo") + addTestFileAndCommit(t, mainRepoPath, "test.txt", "test content") + + // Create a worktree + worktreePath := filepath.Join(filepath.Dir(mainRepoPath), "worktree") + err := exec.Command("git", "-C", mainRepoPath, "worktree", "add", worktreePath, "-b", "worktree-branch").Run() + assert.NoError(t, err) + + // Verify .git is a file in the worktree + gitPath := filepath.Join(worktreePath, ".git") + info, err := os.Stat(gitPath) + assert.NoError(t, err) + assert.False(t, info.IsDir(), ".git should be a file in a worktree") + + // Test resolveGitDir + gitDir, err := resolveGitDir(worktreePath) + assert.NoError(t, err) + assert.NotEqual(t, gitPath, gitDir, "resolved git dir should be different from .git file path") + + // Verify the resolved path is a valid git directory (should contain index) + indexPath := filepath.Join(gitDir, "index") + _, err = os.Stat(indexPath) + assert.NoError(t, err, "resolved git dir should contain index file") + }) + + t.Run("nonexistent repository", func(t *testing.T) { + _, err := resolveGitDir("/nonexistent/path") + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to stat .git") + }) + + t.Run("invalid .git file content", func(t *testing.T) { + tempDir := t.TempDir() + gitPath := filepath.Join(tempDir, ".git") + + // Create an invalid .git file (not starting with "gitdir: ") + err := os.WriteFile(gitPath, []byte("invalid content"), 0644) + assert.NoError(t, err) + + _, err = resolveGitDir(tempDir) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid .git file format") + }) +} + +func TestPrepareRepoWithWorktree(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // Create main repository with staged changes + mainRepoPath := setupTestRepo(t, "main-repo-worktree") + addTestFileAndCommit(t, mainRepoPath, "test.txt", "initial content") + + // Create a worktree + worktreePath := filepath.Join(filepath.Dir(mainRepoPath), "test-worktree") + err := exec.Command("git", "-C", mainRepoPath, "worktree", "add", worktreePath, "-b", "worktree-branch").Run() + assert.NoError(t, err) + + // Stage some changes in the worktree + testFile := filepath.Join(worktreePath, "test.txt") + assert.NoError(t, os.WriteFile(testFile, []byte("modified content in worktree"), 0644)) + assert.NoError(t, exec.Command("git", "-C", worktreePath, "add", "test.txt").Run()) + + // Verify staged changes exist in worktree + output, err := exec.Command("git", "-C", worktreePath, "diff", "--cached").Output() + assert.NoError(t, err) + assert.Contains(t, string(output), "modified content in worktree", "Staged changes should exist in worktree") + + t.Run("PrepareRepo should work with git worktree", func(t *testing.T) { + fileURI := "file://" + worktreePath + preparedPath, isRemote, err := PrepareRepo(ctx, fileURI, "", false, false) + + assert.NoError(t, err, "PrepareRepo should succeed with git worktree") + assert.False(t, isRemote) + assert.NotEmpty(t, preparedPath) + + defer func() { _ = os.RemoveAll(preparedPath) }() + + // Verify the cloned repo has the staged changes preserved + stagedOutput, err := exec.Command("git", "-C", preparedPath, "diff", "--cached").Output() + assert.NoError(t, err, "git diff --cached should succeed in prepared repo") + assert.Contains(t, string(stagedOutput), "modified content in worktree", + "Staged changes should be preserved when cloning from worktree") + }) +} + func TestNormalizeFileURI(t *testing.T) { tests := []struct { name string @@ -941,19 +1366,9 @@ func TestNormalizeFileURI(t *testing.T) { func TestPrepareRepoWithNormalization(t *testing.T) { repoPath := setupTestRepo(t, "test-repo") + t.Chdir(repoPath) addTestFileAndCommit(t, repoPath, "test.txt", "test content") - originalDir, err := os.Getwd() - if err != nil { - t.Fatalf("failed to get current directory: %v", err) - } - defer func() { - if err := os.Chdir(originalDir); err != nil { - t.Fatalf("failed to restore original directory: %v", err) - } - }() - - // Test absolute paths first (without changing directory) absoluteTests := []struct { name string uri string @@ -986,16 +1401,11 @@ func TestPrepareRepoWithNormalization(t *testing.T) { } if path != repoPath && path != "" { - os.RemoveAll(path) + _ = os.RemoveAll(path) } }) } - // Now change to temp directory for relative path tests - if err := os.Chdir(repoPath); err != nil { - t.Fatalf("failed to change to temp directory: %v", err) - } - relativeTests := []struct { name string uri string @@ -1028,19 +1438,19 @@ func TestPrepareRepoWithNormalization(t *testing.T) { } if path != repoPath && path != "" { - os.RemoveAll(path) + _ = os.RemoveAll(path) } }) } } func TestPrepareRepoWithNormalizationBare(t *testing.T) { - t.Parallel() + tempDir := t.TempDir() + t.Chdir(tempDir) workingRepoPath := setupTestRepo(t, "working-repo-original") addTestFileAndCommit(t, workingRepoPath, "test.txt", "test content") - tempDir := t.TempDir() bareRepoPath := filepath.Join(tempDir, "bare-repo") assert.NoError(t, exec.Command("git", "clone", workingRepoPath, bareRepoPath, "--bare").Run()) @@ -1049,17 +1459,6 @@ func TestPrepareRepoWithNormalizationBare(t *testing.T) { _, err = os.Stat(filepath.Join(bareRepoPath, "HEAD")) assert.NoError(t, err, "Bare repository should have HEAD file") - originalDir, err := os.Getwd() - if err != nil { - t.Fatalf("failed to get current directory: %v", err) - } - defer func() { - if err := os.Chdir(originalDir); err != nil { - t.Fatalf("failed to restore original directory: %v", err) - } - }() - - // Test absolute paths first (without changing directory) absoluteTests := []struct { name string uri string @@ -1104,16 +1503,11 @@ func TestPrepareRepoWithNormalizationBare(t *testing.T) { } if path != bareRepoPath && path != "" { - os.RemoveAll(path) + _ = os.RemoveAll(path) } }) } - // Now change to temp directory for relative path tests - if err := os.Chdir(tempDir); err != nil { - t.Fatalf("failed to change to temp directory: %v", err) - } - relativeTests := []struct { name string uri string @@ -1158,8 +1552,775 @@ func TestPrepareRepoWithNormalizationBare(t *testing.T) { } if path != bareRepoPath && path != "" { - os.RemoveAll(path) + _ = os.RemoveAll(path) } }) } } + +// TestGitChunk_LongLine verifies that files containing lines longer than +// bufio's default 64 KB token limit are still scanned. Before the fix, +// bufio.Scanner would silently stop on the first oversized line and produce +// zero chunks for that file. +func TestGitChunk_LongLine(t *testing.T) { + t.Parallel() + ctx := context.Background() + + repoPath := setupTestRepo(t, "long-line-repo") + + // Build a single line that is 100 KB — well above the old 64 KB cap. + longLine := strings.Repeat("a", 100*1024) + addTestFileAndCommit(t, repoPath, "long_line.txt", longLine) + + conn, err := anypb.New(&sourcespb.Git{ + Credential: &sourcespb.Git_Unauthenticated{ + Unauthenticated: &credentialspb.Unauthenticated{}, + }, + Repositories: []string{"file://" + repoPath}, + }) + assert.NoError(t, err) + + s := Source{} + assert.NoError(t, s.Init(ctx, "test long line", 0, 0, false, conn, 1)) + + chunksCh := make(chan *sources.Chunk, 64) + var count int + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for range chunksCh { + count++ + } + }() + + assert.NoError(t, s.Chunks(ctx, chunksCh)) + close(chunksCh) + wg.Wait() + // ensure the goroutine has finished writing to count before we read it + // one chunk for the commit/file metadata, and at least one chunk for the file content + assert.Equal(t, 2, count, "expected two chunks from a file with a 100 KB line") +} + +func TestGitLowMemoryScan(t *testing.T) { + feature.UseGitLowMemoryScan.Store(true) + t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + thisrepo := &sourcespb.Git{ + Directories: []string{"../../../"}, + Credential: &sourcespb.Git_Unauthenticated{ + Unauthenticated: &credentialspb.Unauthenticated{}, + }, + } + wantChunk := &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + SourceName: "this repo, low memory", + SourceVerify: false, + } + + s := Source{} + conn, err := anypb.New(thisrepo) + if err != nil { + t.Fatal(err) + } + + err = s.Init(ctx, "this repo, low memory", 0, 0, false, conn, 1) + if err != nil { + t.Errorf("Source.Init() error = %v", err) + return + } + + chunksCh := make(chan *sources.Chunk, 1) + go func() { + assert.NoError(t, s.Chunks(ctx, chunksCh)) + }() + + gotChunk := <-chunksCh + gotChunk.Data = nil + // Commits don't come in a deterministic order, so remove metadata comparison + gotChunk.SourceMetadata = nil + if diff := pretty.Compare(gotChunk, wantChunk); diff != "" { + t.Errorf("Source.Chunks() UseGitLowMemoryScan diff: (-got +want)\n%s", diff) + t.Errorf("Data: %s", string(gotChunk.Data)) + } +} + +// Planted secrets for the base..head fixture. Unverifiable on purpose; the +// scan runs with verification off and we only assert they were chunked. +const ( + fixtureAWSKey = "AKIAXYZDQCEN4B6JSJQI" + fixtureGitHubToken = "ghp_a1B2c3D4e5F6g7H8i9J0kLmNoPqRsTuVwXyZ" +) + +// mergedBaseFixture is a repository whose feature branch merged its base in, +// the shape reported in INT-1054 / CSM-2357: +// +// F: feature work after merge <- head +// M: merge main into feature +// |\ +// | C: newer base work <- base (and the merge-base of main/feature) +// | B: base work +// E: more feature work <- GitHub token +// D: feature work <- AWS key +// |/ +// A: common ancestor +// +// git log C..F is F M E D. With the customer's dates (D, E older than C) the +// pre-fix scanner stopped at C and never reached E or D. +type mergedBaseFixture struct { + path string + sha map[string]string // commit letter -> full hash +} + +// buildMergedBaseFixture creates the repository above with pinned committer +// dates so the ordering `git log` produces is deterministic. When +// branchNewerThanBase is true the topology is identical but D and E carry +// dates after C, which is the case the pre-fix code happened to get right; +// the fix must produce the same commit set either way. withUnreachableBase +// adds G on main after the merge, so that a base of G is not an ancestor of +// head, the shape GitHub Actions produce via pull_request.base.sha. +func buildMergedBaseFixture(t *testing.T, branchNewerThanBase, withUnreachableBase bool) mergedBaseFixture { + t.Helper() + f := mergedBaseFixture{path: setupTestRepo(t, "merged-base"), sha: map[string]string{}} + git := func(args ...string) { + t.Helper() + out, err := exec.Command("git", append([]string{"-C", f.path}, args...)...).CombinedOutput() + assert.NoError(t, err, "git %v: %s", args, out) + } + appendFile := func(name string, lines ...string) { + t.Helper() + fh, err := os.OpenFile(filepath.Join(f.path, name), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + assert.NoError(t, err) + _, err = fh.WriteString(strings.Join(lines, "\n") + "\n") + assert.NoError(t, err) + assert.NoError(t, fh.Close()) + git("add", name) + } + // commit pins both dates so hashes and log ordering are reproducible. + commit := func(letter, date, msg string) { + t.Helper() + cmd := exec.Command("git", "-C", f.path, "commit", "-q", "-m", msg) + cmd.Env = append(os.Environ(), "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date) + out, err := cmd.CombinedOutput() + assert.NoError(t, err, "commit %s: %s", letter, out) + sha, err := exec.Command("git", "-C", f.path, "rev-parse", "HEAD").Output() + assert.NoError(t, err) + f.sha[letter] = strings.TrimSpace(string(sha)) + } + + // Customer dates: branch work predates the base work it later merges in. + dates := map[string]string{ + "A": "2026-01-01T00:00:00Z", + "D": "2026-01-02T00:00:00Z", "E": "2026-01-03T00:00:00Z", + "B": "2026-01-04T00:00:00Z", "C": "2026-01-05T00:00:00Z", + "M": "2026-01-06T00:00:00Z", "F": "2026-01-07T00:00:00Z", "G": "2026-01-08T00:00:00Z", + } + if branchNewerThanBase { + dates["B"], dates["C"] = "2026-01-02T00:00:00Z", "2026-01-03T00:00:00Z" + dates["D"], dates["E"] = "2026-01-04T00:00:00Z", "2026-01-05T00:00:00Z" + } + + git("switch", "-q", "-c", "main") + appendFile("README.md", "A") + commit("A", dates["A"], "A: common ancestor") + + git("switch", "-q", "-c", "feature") + appendFile("feature.txt", "D", + "aws_access_key_id = "+fixtureAWSKey, + "aws_secret_access_key = Tg0pz8Jii8hkLx4+PnUisM8GmKs3a2DK+9qz/lie") + commit("D", dates["D"], "D: feature work") + appendFile("feature.txt", "E", "github_token = "+fixtureGitHubToken) + commit("E", dates["E"], "E: more feature work") + + git("switch", "-q", "main") + appendFile("base.txt", "B") + commit("B", dates["B"], "B: base work") + appendFile("base.txt", "C") + commit("C", dates["C"], "C: newer base work") + + git("switch", "-q", "feature") + { + cmd := exec.Command("git", "-C", f.path, "merge", "-q", "--no-ff", "main", "-m", "M: merge main into feature") + cmd.Env = append(os.Environ(), "GIT_AUTHOR_DATE="+dates["M"], "GIT_COMMITTER_DATE="+dates["M"]) + out, err := cmd.CombinedOutput() + assert.NoError(t, err, "merge: %s", out) + sha, err := exec.Command("git", "-C", f.path, "rev-parse", "HEAD").Output() + assert.NoError(t, err) + f.sha["M"] = strings.TrimSpace(string(sha)) + } + appendFile("feature.txt", "F", "github_token = "+fixtureGitHubToken) + commit("F", dates["F"], "F: feature work after merge") + + if withUnreachableBase { + git("switch", "-q", "main") + appendFile("base.txt", "G") + commit("G", dates["G"], "G: base work after the merge") + git("switch", "-q", "feature") + } + return f +} + +// scanFixtureCommits runs Git.ScanRepo over the fixture with the given +// base/head and returns the set of commit hashes that produced chunks plus the +// concatenated chunk data, so callers can assert both coverage and content. +func scanFixtureCommits(t *testing.T, f mergedBaseFixture, base, head string) (map[string]bool, string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + got, data, err := scanRepoRange(ctx, t, f.path, base, head) + assert.NoError(t, err) + return got, data +} + +// scanRepoRange runs a base..head diff scan over the repository at repoPath and +// reports the commits that produced chunks, the concatenated chunk data, and +// the scan error. Callers that expect a failing scan assert on the error. +func scanRepoRange(ctx context.Context, t *testing.T, repoPath, base, head string) (map[string]bool, string, error) { + t.Helper() + + // Open through the package's own wrapper so the test takes the same + // path real callers do (bare detection, .git discovery). + repo, err := RepoFromPath(repoPath) + if err != nil { + return nil, "", err + } + + g := NewGit(&Config{ + SourceName: "range fixture", + SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT, + Concurrency: 1, + SourceMetadataFunc: func(info SourceMetadataInfo) *source_metadatapb.MetaData { + return &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{Git: &source_metadatapb.Git{Commit: info.Commit, File: info.File}}, + } + }, + }) + + chunksCh := make(chan *sources.Chunk, 64) + scanErr := make(chan error, 1) + go func() { + defer close(chunksCh) + scanErr <- g.ScanRepo(ctx, repo, repoPath, NewScanOptions(ScanOptionBaseHash(base), ScanOptionHeadCommit(head)), sources.ChanReporter{Ch: chunksCh}) + }() + + got := map[string]bool{} + var data strings.Builder + for c := range chunksCh { + got[c.SourceMetadata.GetGit().GetCommit()] = true + data.Write(c.Data) + } + return got, data.String(), <-scanErr +} + +// TestScanRepo_BaseMergedIntoHead is the regression test for INT-1054: a +// diff scan must cover exactly `git log base..head` regardless of commit +// dates or whether base is reachable from head. +func TestScanRepo_BaseMergedIntoHead(t *testing.T) { + // Every commit on the feature side of the range, i.e. git log C..F. + wantScanned := []string{"F", "M", "E", "D"} + // The merged-in base work and the common ancestor must never be scanned. + wantSkipped := []string{"A", "B", "C"} + + cases := []struct { + name string + branchNewerThanBase bool + unreachableBase bool + base string // commit letter passed as --since-commit + }{ + { + // The customer's reproducer: base is the main tip that was merged in, + // and the branch commits predate it. + name: "base is the merged-in main tip and older branch commits are skipped", + base: "C", + }, + { + // GitHub Action pull_request path: base.sha has moved past the merge, + // so normalizeConfig resolves it to the merge-base C. Must match case 1. + name: "base is unreachable from head", + unreachableBase: true, + base: "G", + }, + { + // Same topology, dates flipped: pins that the result is a function of + // the graph, not of committer dates. + name: "branch commits newer than the merged-in base tip", + branchNewerThanBase: true, + base: "C", + }, + } + + // Both parser strategies build their `git log` from the same args, so both + // must agree. + for _, lowMemory := range []bool{false, true} { + mode := "default" + if lowMemory { + mode = "low-memory" + } + t.Run(mode, func(t *testing.T) { + feature.UseGitLowMemoryScan.Store(lowMemory) + t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) }) + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := buildMergedBaseFixture(t, tc.branchNewerThanBase, tc.unreachableBase) + got, data := scanFixtureCommits(t, f, f.sha[tc.base], f.sha["F"]) + + for _, letter := range wantScanned { + assert.True(t, got[f.sha[letter]], "commit %s (%s) should have been scanned; got %v", letter, f.sha[letter][:7], got) + } + for _, letter := range wantSkipped { + assert.False(t, got[f.sha[letter]], "commit %s (%s) is reachable from base and must not be scanned", letter, f.sha[letter][:7]) + } + // The secrets live in D and E, the commits the pre-fix code dropped. + assert.Contains(t, data, fixtureAWSKey, "AWS key from commit D missing from scanned data") + assert.Contains(t, data, fixtureGitHubToken, "GitHub token from commit E/F missing from scanned data") + }) + } + }) + } +} + +// TestNormalizeConfig_BaseWithoutHead pins what normalizeConfig does and does +// not do with a base and no head: it resolves the base to a hash and leaves the +// head empty. The parser then walks --all ^base (see gitparse.Parser.RepoPath). +// normalizeConfig must not invent a head, because that would route the +// base-only shape through MergeBase, which go-git cannot compute across the +// graft of a shallow clone, including the --shallow-since clone +// prepareRepoSinceCommit makes for exactly this shape. +func TestNormalizeConfig_BaseWithoutHead(t *testing.T) { + // main: A; feature: A -> B, checked out. + path := setupTestRepo(t, "base-without-head") + addTestFileAndCommit(t, path, "a.txt", "a\n") + runGit(t, path, "branch", "-M", "main") + shaA := gitRevParse(t, path, "HEAD") + runGit(t, path, "switch", "-q", "-c", "feature") + addTestFileAndCommit(t, path, "b.txt", "b\n") + shaB := gitRevParse(t, path, "HEAD") + + repo, err := RepoFromPath(path) + assert.NoError(t, err) + + tests := []struct { + name string + base, head string + wantBase string + wantHead string + }{ + { + // The base is resolved to a hash; the head stays empty and no merge + // base is computed. The parser turns this into --all ^main. + name: "base ref and no head leaves head empty", + base: "main", wantBase: shaA, wantHead: "", + }, + { + // The pre-commit hook passes HEAD as both ends (main.go). Both + // resolve to the same hash and MergeBase short-circuits without + // walking history, so the empty HEAD..HEAD range is safe on a + // shallow local clone too. + name: "base HEAD and head HEAD resolve to the same commit", + base: "HEAD", head: "HEAD", wantBase: shaB, wantHead: shaB, + }, + { + // Full-history scans set neither end and must stay that way; an + // implicit head here would silently narrow --all to one branch. + name: "no base leaves head empty", + base: "", wantBase: "", wantHead: "", + }, + { + name: "head without base is resolved but gets no base", + head: "feature", wantBase: "", wantHead: shaB, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := NewScanOptions(ScanOptionBaseHash(tt.base), ScanOptionHeadCommit(tt.head)) + assert.NoError(t, normalizeConfig(opts, repo)) + assert.Equal(t, tt.wantBase, opts.BaseHash, "BaseHash") + assert.Equal(t, tt.wantHead, opts.HeadHash, "HeadHash") + }) + } +} + +// TestScanRepo_BaseWithoutHead runs the base-without-head shape end to end. +// `--since-commit X` with no `--branch` covers every commit since X on any +// ref (`git log --all ^X`), not just the checked-out branch: users scanning a +// mirror or a remote URL rely on that to catch secrets on side branches. The +// merged-base fixture gets an extra branch with its own planted secret that is +// not reachable from the checked-out feature branch; a base-only scan must +// find it, and the pre-commit hook, which passes HEAD as both ends, must not. +func TestScanRepo_BaseWithoutHead(t *testing.T) { + const ( + sideSecret = "ghp_SideBranchTokenThatBaseOnlyMustCatch000" + stagedSecret = "ghp_StagedTokenThatPreCommitMustStillCatch0" + ) + + // Fixture ends checked out on feature at F. Add the side branch off the + // common ancestor and come back to feature so HEAD is F. + build := func(t *testing.T) mergedBaseFixture { + f := buildMergedBaseFixture(t, false, false) + runGit(t, f.path, "switch", "-q", "-c", "side", f.sha["A"]) + addTestFileAndCommit(t, f.path, "side.txt", "github_token = "+sideSecret+"\n") + f.sha["X"] = gitRevParse(t, f.path, "HEAD") + runGit(t, f.path, "switch", "-q", "feature") + return f + } + + // assertAllSinceC checks the --all ^C set: everything on feature after the + // merge base plus the side branch, never the base's own history. + assertAllSinceC := func(t *testing.T, f mergedBaseFixture, got map[string]bool, data string) { + t.Helper() + for _, letter := range []string{"F", "M", "E", "D", "X"} { + assert.True(t, got[f.sha[letter]], "commit %s should have been scanned; got %v", letter, got) + } + for _, letter := range []string{"A", "B", "C"} { + assert.False(t, got[f.sha[letter]], "commit %s is reachable from C and must not be scanned", letter) + } + assert.Contains(t, data, sideSecret, "a base-only scan must cover branches other than the checked-out one") + } + + for _, lowMemory := range []bool{false, true} { + mode := "default" + if lowMemory { + mode = "low-memory" + } + t.Run(mode, func(t *testing.T) { + feature.UseGitLowMemoryScan.Store(lowMemory) + t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) }) + + t.Run("base only scans every ref since base", func(t *testing.T) { + f := build(t) + got, data := scanFixtureCommits(t, f, f.sha["C"], "") + assertAllSinceC(t, f, got, data) + }) + + t.Run("explicit head narrows to one branch", func(t *testing.T) { + f := build(t) + got, data := scanFixtureCommits(t, f, f.sha["C"], f.sha["F"]) + + // git log C..F: the side branch is not reachable from F. + for _, letter := range []string{"F", "M", "E", "D"} { + assert.True(t, got[f.sha[letter]], "commit %s should have been scanned; got %v", letter, got) + } + for _, letter := range []string{"A", "B", "C", "X"} { + assert.False(t, got[f.sha[letter]], "commit %s is outside C..F and must not be scanned", letter) + } + assert.NotContains(t, data, sideSecret) + }) + + // The history from the PR review, where the old --all walk that + // stopped at the base happened to cover the side branch: + // + // o1 - o2 other + // / + // a - b - c main <- HEAD + // + // --since-commit a with no --branch must still scan o1 and o2. + t.Run("commits only on other branches are scanned", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + path := setupTestRepo(t, "cross-ref") + sha := map[string]string{} + commit := func(name string) { + addTestFileAndCommit(t, path, name+".txt", name+"\n") + sha[name] = gitRevParse(t, path, "HEAD") + } + commit("a") + runGit(t, path, "branch", "-M", "main") + runGit(t, path, "switch", "-q", "-c", "other") + commit("o1") + commit("o2") + runGit(t, path, "switch", "-q", "main") + commit("b") + commit("c") + + got, _, err := scanRepoRange(ctx, t, path, sha["a"], "") + assert.NoError(t, err) + for _, name := range []string{"b", "c", "o1", "o2"} { + assert.True(t, got[sha[name]], "commit %s should have been scanned; got %v", name, got) + } + assert.False(t, got[sha["a"]], "the base is excluded from its own range") + + // With --branch main the same base covers main only. + got, _, err = scanRepoRange(ctx, t, path, sha["a"], "main") + assert.NoError(t, err) + for _, name := range []string{"b", "c"} { + assert.True(t, got[sha[name]], "commit %s should have been scanned; got %v", name, got) + } + for _, name := range []string{"a", "o1", "o2"} { + assert.False(t, got[sha[name]], "commit %s is outside a..main and must not be scanned", name) + } + }) + + // main.go's pre-commit override sets both ends to HEAD, so the + // commit range is empty and only staged changes are scanned. + t.Run("pre-commit shape scans only staged changes", func(t *testing.T) { + f := build(t) + assert.NoError(t, os.WriteFile(filepath.Join(f.path, "staged.txt"), []byte("github_token = "+stagedSecret+"\n"), 0o644)) + runGit(t, f.path, "add", "staged.txt") + + got, data := scanFixtureCommits(t, f, "HEAD", "HEAD") + + // HEAD..HEAD is empty, so no commit in the repository may produce + // chunks. Staged chunks carry no commit hash and are asserted on + // through the data instead. + for letter, sha := range f.sha { + assert.False(t, got[sha], "commit %s was scanned but HEAD..HEAD is empty; got %v", letter, got) + } + assert.Contains(t, data, stagedSecret, "staged changes must still be scanned") + assert.NotContains(t, data, sideSecret, "a commit on another branch leaked into the hook scan") + }) + + // Remote URLs are cloned with --mirror, so production base-only scans + // run against a bare repository and every ref the mirror carries is + // in scope. + t.Run("bare mirror clone scans every ref since base", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + f := build(t) + mirror := filepath.Join(t.TempDir(), "mirror.git") + runGit(t, "", "clone", "-q", "--mirror", "file://"+f.path, mirror) + assert.True(t, isRepoBare(mirror), "fixture is not bare") + + got, data, err := scanRepoRange(ctx, t, mirror, f.sha["C"], "") + assert.NoError(t, err) + assertAllSinceC(t, f, got, data) + }) + }) + } +} + +// TestScanRepo_SwappedRangeEnds pins the behavior when the base is a descendant +// of the head, e.g. a CI job that passes its arguments in the wrong order. The +// range base..head is empty, so nothing is scanned and the scan succeeds; it +// must not widen to anything else. ScanCommits logs the empty range at Info so +// the case is visible at the default log level without failing the pre-commit +// hook, whose HEAD..HEAD range is empty by design. +func TestScanRepo_SwappedRangeEnds(t *testing.T) { + f := buildMergedBaseFixture(t, false, false) + + // base F is a descendant of head C: git log C ^F is empty. + got, data := scanFixtureCommits(t, f, f.sha["F"], f.sha["C"]) + + assert.Empty(t, got, "an empty range must not scan any commit; got %v", got) + assert.NotContains(t, data, fixtureAWSKey) + assert.NotContains(t, data, fixtureGitHubToken) +} + +// TestScanRepo_BaseNotUsable pins the behavior when a diff scan is given a base +// it cannot turn into a commit in the repository. Every case must fail the scan: +// the tempting alternative, degrading to a full-history scan, reports success +// for a scan that covered a different range than the user asked for, which is +// the class of silent miss INT-1054 is about. +func TestScanRepo_BaseNotUsable(t *testing.T) { + // A syntactically valid hash that is not in any fixture below. + const absentHash = "1111111111111111111111111111111111111111" + + tests := []struct { + name string + // setup returns the repo path plus the base and head to scan. + setup func(t *testing.T) (repoPath, base, head string) + wantErr string + }{ + { + // A branch name that does not exist, e.g. a CI job passing a deleted + // or misspelled base branch. + name: "base ref does not exist", + setup: func(t *testing.T) (string, string, string) { + path := setupTestRepo(t, "unknown-ref") + addTestFileAndCommit(t, path, "a.txt", "a\n") + return path, "no-such-branch", gitRevParse(t, path, "HEAD") + }, + wantErr: "unable to resolve ref", + }, + { + // A well-formed hash for an object the repository does not have. This + // is what an externally shallow checkout looks like when the base is + // older than the fetch depth. + name: "base hash is not in the repository", + setup: func(t *testing.T) (string, string, string) { + path := setupTestRepo(t, "absent-hash") + addTestFileAndCommit(t, path, "a.txt", "a\n") + return path, absentHash, gitRevParse(t, path, "HEAD") + }, + wantErr: "unable to resolve commit", + }, + { + // Two root commits: both refs resolve, but they share no history, so + // there is no range between them to scan. + name: "base and head have no common ancestor", + setup: func(t *testing.T) (string, string, string) { + path := setupTestRepo(t, "unrelated") + addTestFileAndCommit(t, path, "a.txt", "a\n") + base := gitRevParse(t, path, "HEAD") + runGit(t, path, "switch", "-q", "--orphan", "unrelated") + addTestFileAndCommit(t, path, "b.txt", "b\n") + return path, base, gitRevParse(t, path, "HEAD") + }, + wantErr: "merge base", + }, + { + // actions/checkout at its default fetch-depth: the clone holds only + // the tip, so a base from before the cutoff is simply absent. + name: "shallow clone does not contain the base", + setup: func(t *testing.T) (string, string, string) { + origin := setupTestRepo(t, "shallow-origin") + addTestFileAndCommit(t, origin, "a.txt", "a\n") + base := gitRevParse(t, origin, "HEAD") + addTestFileAndCommit(t, origin, "b.txt", "b\n") + + // --depth needs a file:// URL; git ignores it for plain local paths. + shallow := filepath.Join(t.TempDir(), "shallow") + runGit(t, "", "clone", "-q", "--depth", "1", "file://"+origin, shallow) + assert.False(t, gitHasObject(t, shallow, base), "fixture is not shallow: base is present") + + return shallow, base, gitRevParse(t, shallow, "HEAD") + }, + wantErr: "unable to resolve commit", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + repoPath, base, head := tt.setup(t) + got, _, err := scanRepoRange(ctx, t, repoPath, base, head) + + assert.ErrorContains(t, err, tt.wantErr) + assert.Empty(t, got, "a scan that cannot resolve its base must not fall back to scanning commits") + }) + } +} + +// TestScanRepo_ShallowClone pins where the graft boundary of a shallow clone +// starts to matter for a diff scan. git itself handles a truncated history +// fine, so the boundary that counts is the one go-git needs to walk in +// normalizeConfig: MergeBase loads a commit's parents before it can recognize +// the commit as the range boundary (bfsCommitIterator.Next), so a base sitting +// on the graft has no resolvable merge base. +// +// That walk only runs when both ends are supplied. With a base alone the +// parser defaults the head to HEAD and git computes the range, so the same +// clone scans cleanly; this is the shape `--since-commit X` without `--branch` +// takes against a GitHub URL, where prepareRepoSinceCommit clones with +// --shallow-since and puts X on the graft. See +// https://github.com/trufflesecurity/trufflehog/issues/4895 for the both-ends +// failure surfacing on GitLab. +func TestScanRepo_ShallowClone(t *testing.T) { + // A fixture deep enough that a clone can keep the base and still cut the + // history off somewhere above it. + newOrigin := func(t *testing.T) string { + origin := setupTestRepo(t, "shallow-origin") + for _, f := range []string{"a.txt", "b.txt", "c.txt", "d.txt"} { + addTestFileAndCommit(t, origin, f, f+"\n") + } + return origin + } + clone := func(t *testing.T, origin, depth string) string { + // --depth needs a file:// URL; git ignores it for plain local paths. + path := filepath.Join(t.TempDir(), "shallow") + runGit(t, "", "clone", "-q", "--depth", depth, "file://"+origin, path) + return path + } + // Depth 2 keeps the tip and the base; the base's parent is cut off, which + // is the shape --shallow-since produces for its own base commit. + graftAtBase := func(t *testing.T) (shallow, head, base string) { + shallow = clone(t, newOrigin(t), "2") + head, base = gitRevParse(t, shallow, "HEAD"), gitRevParse(t, shallow, "HEAD~1") + assert.False(t, gitHasObject(t, shallow, base+"^"), "fixture is wrong: the base's parent is present") + return shallow, head, base + } + + // Both parser strategies build their `git log` from the same args, so both + // must agree. + for _, lowMemory := range []bool{false, true} { + mode := "default" + if lowMemory { + mode = "low-memory" + } + t.Run(mode, func(t *testing.T) { + feature.UseGitLowMemoryScan.Store(lowMemory) + t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) }) + + t.Run("base is the graft boundary", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + shallow, head, base := graftAtBase(t) + + got, _, err := scanRepoRange(ctx, t, shallow, base, head) + + assert.ErrorContains(t, err, "unable to resolve merge base") + assert.Empty(t, got, "an unresolvable merge base must fail the scan, not scan an arbitrary range") + }) + + t.Run("base is the graft boundary with implicit head", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Same clone, no head: normalizeConfig must not invent a head + // and run MergeBase, or the --shallow-since path would fail + // exactly where it used to work. git walks --all ^base, and the + // clone's only ref is the tip. + shallow, head, base := graftAtBase(t) + + got, _, err := scanRepoRange(ctx, t, shallow, base, "") + + assert.NoError(t, err) + assert.True(t, got[head], "the one commit since base should have been scanned; got %v", got) + assert.False(t, got[base], "the base is excluded from its own range") + }) + + t.Run("base is above the graft boundary", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // One more commit of depth is all it takes: the base's parent is + // present, so the merge base resolves and git scans the range over + // a history it cannot fully walk. + shallow := clone(t, newOrigin(t), "3") + head, base := gitRevParse(t, shallow, "HEAD"), gitRevParse(t, shallow, "HEAD~1") + assert.True(t, gitHasObject(t, shallow, base+"^"), "fixture is wrong: the base's parent is missing") + + got, _, err := scanRepoRange(ctx, t, shallow, base, head) + + assert.NoError(t, err) + assert.True(t, got[head], "the one commit in base..head should have been scanned; got %v", got) + assert.False(t, got[base], "the base is excluded from its own range") + }) + }) + } +} + +// runGit runs a git command, failing the test on error. An empty repoPath runs +// git outside any repository, which clone needs. +func runGit(t *testing.T, repoPath string, args ...string) { + t.Helper() + if repoPath != "" { + args = append([]string{"-C", repoPath}, args...) + } + out, err := exec.Command("git", args...).CombinedOutput() + assert.NoError(t, err, "git %v: %s", args, out) +} + +func gitRevParse(t *testing.T, repoPath, rev string) string { + t.Helper() + out, err := exec.Command("git", "-C", repoPath, "rev-parse", rev).Output() + assert.NoError(t, err) + return strings.TrimSpace(string(out)) +} + +// gitHasObject reports whether the repository holds the commit named by rev, +// used to confirm where a shallow fixture's graft boundary actually landed +// before asserting on behavior that depends on it. +func gitHasObject(t *testing.T, repoPath, rev string) bool { + t.Helper() + return exec.Command("git", "-C", repoPath, "cat-file", "-e", rev+"^{commit}").Run() == nil +} diff --git a/pkg/sources/git/metrics.go b/pkg/sources/git/metrics.go index 666440a46144..263ebc7af078 100644 --- a/pkg/sources/git/metrics.go +++ b/pkg/sources/git/metrics.go @@ -105,14 +105,27 @@ func ClassifyCloneError(errMsg string) string { strings.Contains(errMsg, "redirect:") && strings.Contains(errMsg, "users/sign_in"): return cloneFailureAuth + // Checked before the generic 403/429 case below: when the git backend + // itself explains a 403/429 with a "remote:" line, that's a permanent + // failure (permission denial, SAML SSO enforcement, org policy, etc.), + // not throttling — the backend only sends an explanatory message once + // the request has been evaluated, whereas secondary-rate-limit responses + // are either a bare 403/429 with no remote explanation or one of the + // small, stable set of known throttling messages. Matching on "any + // unrecognized remote: explanation" avoids having to enumerate every + // provider's denial wording, which changes per provider and per feature + // (SSO, fine-grained tokens, GitHub Apps, ...). + case isRemotePermissionDenial(errMsg): + return cloneFailurePermission + case strings.Contains(errMsg, "The requested URL returned error: 429") || + strings.Contains(errMsg, "The requested URL returned error: 403") || strings.Contains(errMsg, "remote: Retry later"): + // A bare "403" during clone (no accompanying auth/permission message) + // matches GitHub/GitLab secondary rate limiting and abuse-detection + // responses, which are documented to return either 403 or 429. return cloneFailureRateLimit - case strings.Contains(errMsg, "The requested URL returned error: 403") || - strings.Contains(errMsg, "remote: You are not allowed to download code from this project"): - return cloneFailurePermission - case strings.Contains(errMsg, "RPC failed") || strings.Contains(errMsg, "unexpected disconnect") || strings.Contains(errMsg, "early EOF") || @@ -128,3 +141,63 @@ func ClassifyCloneError(errMsg string) string { return cloneFailureOther } } + +// knownRateLimitRemoteMessages are the (small, stable) set of ways GitHub and +// GitLab phrase an actual throttling response in a "remote:" line. Any other +// "remote:" explanation accompanying a 403/429 is treated as a permanent +// failure rather than added here, since provider denial wording (permission, +// SSO, org policy, ...) varies far more than throttling wording does. +var knownRateLimitRemoteMessages = []string{ + "retry later", + "secondary rate limit", + "rate limit exceeded", + "abuse detection", +} + +// knownBenignRemoteProgressMessages are the small, standardized set of +// progress lines git-upload-pack prints to "remote:" during a normal clone +// (e.g. "remote: Counting objects: 100% (5/5), done."). These can appear +// ahead of an unrelated 403/429 or network failure, so they must not be +// mistaken for a denial explanation. +var knownBenignRemoteProgressMessages = []string{ + "enumerating objects", + "counting objects", + "compressing objects", + "writing objects", + "resolving deltas", + "total ", +} + +// knownBenignRemoteMessages is the combined set of "remote:" line contents +// that must NOT be treated as a permission denial: known throttling messages +// and known clone-progress output. +var knownBenignRemoteMessages = append(append([]string{}, knownRateLimitRemoteMessages...), knownBenignRemoteProgressMessages...) + +// isRemotePermissionDenial reports whether errMsg contains a server-side +// "remote:" line explaining a 403/429 as something other than throttling or +// normal clone progress output, as opposed to a bare curl-level 403/429 with +// no explanation (or an explicit rate-limit message), which may just be +// transient rate limiting. +func isRemotePermissionDenial(errMsg string) bool { + if !strings.Contains(errMsg, "403") && !strings.Contains(errMsg, "429") { + return false + } + for _, line := range strings.Split(errMsg, "\n") { + idx := strings.Index(line, "remote:") + if idx == -1 { + continue + } + remoteMsg := strings.ToLower(line[idx:]) + benign := false + for _, marker := range knownBenignRemoteMessages { + if strings.Contains(remoteMsg, marker) { + benign = true + break + } + } + if !benign { + return true + } + } + return false +} diff --git a/pkg/sources/github/README.md b/pkg/sources/github/README.md new file mode 100644 index 000000000000..805d8309e6ef --- /dev/null +++ b/pkg/sources/github/README.md @@ -0,0 +1,344 @@ +# GitHub Source + +## Overview + +The GitHub source lets TruffleHog scan GitHub repositories, gists, wikis, issues, pull requests, and their comments for secrets, credentials, and sensitive data. It uses the GitHub API to find repositories, then clones and scans each one with the git source underneath. + +## GitHub Fundamentals + +### What does this source scan? + +This source finds repositories through the GitHub API (by organization, user, or an explicit list), then clones and scans each repository the same way the git source does. On top of that, it can also scan a repository's wiki, and the text of issues, pull requests, gists, and their comments through the API directly. + +### Key GitHub Terminology + +| Term | Description | +|------|-------------| +| **Organization** | A GitHub account that holds repositories owned by a team or company | +| **Repository** | A single project, holding its git history | +| **Fork** | A copy of another repository, made under a different owner | +| **Gist** | A small, standalone snippet of code or text, with its own git history | +| **GitHub App** | An identity registered with GitHub that can be installed into an organization or account to access its repositories | +| **Installation** | A GitHub App added to a specific organization or account, with access scoped to that installation | +| **GitHub Enterprise Server (GHES)** | A self hosted copy of GitHub, running on the customer's own infrastructure | +| **GHE.com** | GitHub Enterprise Cloud with data residency, hosted by GitHub on a dedicated subdomain | + +## Features + +- **Multiple Authentication Methods**: Personal access token, GitHub App, username and password, or unauthenticated for public repositories +- **Multiple Scan Targets**: Scan by organization, by user, or by an explicit list of repositories +- **Repository Filtering**: Include or exclude repositories in an organization or user scan using glob patterns, exclude forks and archived repositories +- **Comment Scanning**: Optionally scan issue, pull request, and gist comments, limited to a number of past days if needed +- **Wiki Scanning**: Optionally scan a repository's wiki alongside its code +- **Self Hosted Support**: Works with GitHub Enterprise Server and GHE.com in addition to github.com +- **Concurrent Processing**: Repositories are cloned and scanned in parallel across a configurable number of workers + +## Configuration + +### Authentication Methods + +#### 1. Personal Access Token + +**CLI Usage:** +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitHub + organizations: + - my-org + token: "ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + name: github-scan + type: SOURCE_TYPE_GITHUB + verify: true +``` + +On the CLI, the token can also be set with the `GITHUB_TOKEN` environment variable instead of `--token`. This does not apply to a YAML config file, since TruffleHog does not read environment variables while loading YAML. Without a token, the CLI falls back to an unauthenticated scan. + +--- + +#### 2. GitHub App + +Uses a GitHub App installed into an organization or account. This method is only available in the YAML config, not the CLI. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitHub + organizations: + - my-org + github_app: + app_id: "123456" + installation_id: "789012" + private_key: | + -----BEGIN RSA PRIVATE KEY----- + ... + -----END RSA PRIVATE KEY----- + name: github-scan + type: SOURCE_TYPE_GITHUB + verify: true +``` + +Set `scan_all_installations: true` to scan every organization and account the app is installed into, instead of a single `installation_id`. Some things still need a fallback installation even with `scan_all_installations` set, such as gists, scanning organization members, or repositories that no installation listing owns. Keep `installation_id` set alongside `scan_all_installations` if you use any of those, otherwise they are skipped with a log message instead of failing the scan. + +--- + +#### 3. Username and Password + +This method is only available in the YAML config, not the CLI. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitHub + organizations: + - my-org + basic_auth: + username: my-username + password: my-password + name: github-scan + type: SOURCE_TYPE_GITHUB + verify: true +``` + +--- + +#### 4. Unauthenticated + +Only works for public repositories, and is heavily rate limited by GitHub. + +**CLI Usage:** +```bash +trufflehog github --org my-org +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitHub + organizations: + - my-org + unauthenticated: {} + name: github-scan + type: SOURCE_TYPE_GITHUB + verify: true +``` + +### Choosing What to Scan + +On the CLI, exactly one of `--org` or `--repo` must be given, the CLI rejects both together. A YAML config file does not have this restriction: `organizations` and `repositories` can both be set at once, and the source scans the explicit repositories plus whatever the organizations enumerate. + +**Scan an Organization or User** + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +The same flag works for a user account, since the source tries an organization lookup first and falls back to a user lookup if that fails. + +**Scan Specific Repositories** + +```bash +trufflehog github --repo https://github.com/my-org/my-repo --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +`--repo` can be repeated. When repositories are given directly, they bypass `--include-repos` and `--exclude-repos` entirely, since those filters only apply to repositories found through an organization or user scan. + +### Filtering What Gets Scanned + +**Include or Exclude Repositories** + +Only applies to an organization or user scan, and cannot be combined with `--repo`. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --include-repos "my-org/prod-*" +``` + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --exclude-repos "my-org/test-*" +``` + +**Include Forks** + +Forks are excluded by default. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --include-forks +``` + +**Exclude Archived Repositories** + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --exclude-archived +``` + +**Include or Exclude Files** + +Both flags take a path to a file that holds one regex per line. Blank lines and lines starting with `#` are ignored. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --include-paths include-patterns.txt +``` + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --exclude-paths exclude-patterns.txt +``` + +**Number of Workers** + +There is no GitHub specific flag for this. Use TruffleHog's global `--concurrency` flag to set how many repositories are cloned and scanned at the same time. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --concurrency 10 +``` + +### Scanning Comments and Wikis + +**Issue and Pull Request Comments** + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --issue-comments --pr-comments +``` + +**Gist Comments** + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --gist-comments +``` + +**Limit Comments to Recent Days** + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --issue-comments --comments-timeframe 30 +``` + +**Repository Wikis** + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --include-wikis +``` + +**Scan Organization Members** + +Also scans the personal repositories and gists of every member of a scanned organization. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --include-members +``` + +**Ignore Gists** + +Gists are only fetched for a user account, not a genuine organization: when `--org` is a username rather than an organization name, when scanning the token owner's own account, or when `--include-members` finds a member. They are included by default in those cases. This flag turns that off. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --ignore-gists +``` + +### Self Hosted GitHub + +Point at a GitHub Enterprise Server or GHE.com instance instead of github.com. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --endpoint https://github.mycompany.com +``` + +### Clone Location and Cleanup + +By default, each repository is cloned into a temporary directory and deleted after scanning. `--clone-path` clones into a given directory instead, and `--no-cleanup` (only valid together with `--clone-path`) keeps the clones on disk afterward instead of deleting them. + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --clone-path /path/to/clones --no-cleanup +``` + +With `--clone-path` set, each repository still gets its own fresh subdirectory created inside it (named `trufflehog--...`), TruffleHog never clones into `--clone-path` itself. Without `--no-cleanup`, that subdirectory is deleted after the repository is scanned, but `--clone-path` itself and anything else already in it is left alone. + +## How Scanning Works + +### Scanning Process + +1. **Enumeration**: Depending on the auth method and target, lists repositories from the given organizations, users, or the explicit repository list, applying the include, exclude, fork, and archived filters +2. **Repository Cloning**: Clones each enumerated repository using the git source, the same way the git source scans a remote repository +3. **Commit Scanning**: Scans every commit in the clone for secrets, the same way the git source does +4. **Wiki Scanning**: If `--include-wikis` is set and a repository has a wiki, clones and scans the wiki as its own repository +5. **Comment Scanning**: If any comment flags are set, fetches issues, pull requests, or gist comments through the GitHub API and scans their text +6. **Progress Tracking**: Keeps track of which repositories have already been scanned, so a scan can be resumed without redoing work + +### What Gets Scanned + +- Every commit in every repository matching the scan target and filters +- Wiki commits, when `--include-wikis` is set and the repository has a wiki +- Issue, pull request, and gist text and comments, when the corresponding flags are set + +### What Doesn't Get Scanned + +- Forked repositories, unless `--include-forks` is set +- Archived repositories, when `--exclude-archived` is set +- Repositories excluded by `--include-repos` or `--exclude-repos`, for an organization or user scan +- Gists, when `--ignore-gists` is set, or when the scan target is a genuine organization rather than a user (gists only surface for user accounts, see the note above) +- Comments older than the configured comments timeframe, when one is set + +## Usage Examples + +### Scanning an Organization + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning Specific Repositories + +```bash +trufflehog github --repo https://github.com/my-org/repo-one --repo https://github.com/my-org/repo-two --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning an Organization Including Comments and Wikis + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --issue-comments --pr-comments --include-wikis +``` + +### Scanning a Public Repository Without Authentication + +```bash +trufflehog github --repo https://github.com/my-org/public-repo +``` + +### Scanning GitHub Enterprise Server + +```bash +trufflehog github --org my-org --token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx --endpoint https://github.mycompany.com +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Authentication failures when connecting to GitHub +**Solution**: Check that the token, GitHub App credentials, or username and password are correct and have permission to read the target repositories. + +--- + +**Issue**: `invalid config: you must specify at least one organization or repository` +**Solution**: Pass either `--org` or `--repo`, exactly one of the two. + +--- + +**Issue**: `invalid config: --include-repos and --exclude-repos only apply to organization or user scans and cannot be used with --repo` +**Solution**: These filters only affect repositories discovered through an organization or user scan. Remove `--repo` or remove the include and exclude filters. + +--- + +**Issue**: Rate limiting during a large scan +**Solution**: Use an authenticated token or GitHub App instead of an unauthenticated scan, since unauthenticated requests have a much lower rate limit. + +--- + +**Issue**: A repository's wiki is not scanned even with `--include-wikis` +**Solution**: GitHub's API can report that a repository has a wiki even when it does not. TruffleHog ignores a wiki clone that comes back as not found. diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 126657cf7f55..9e99e4e1fdfc 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -18,28 +18,106 @@ import ( const ( cloudV3Endpoint = "https://api.github.com" cloudGraphqlEndpoint = "https://api.github.com/graphql" // https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#the-graphql-endpoint + + gheCloudSuffix = ".ghe.com" ) // Connector abstracts over the authenticated ways to interact with GitHub: cloning and API operations. type Connector interface { // APIClient returns a configured GitHub client that can be used for GitHub API operations. APIClient() *github.Client + // APIClientForRepo returns a GitHub API client scoped to the repository URL when needed. + APIClientForRepo(repoURL string) (*github.Client, error) // GraphQLClient returns a client that can be used for GraphQL operations. GraphQLClient() *githubv4.Client + // GraphQLClientForRepo returns a GitHub GraphQL client scoped to the repository URL when needed. + GraphQLClientForRepo(ctx context.Context, repoURL string) (*githubv4.Client, error) // Clone clones a repository using the configured authentication information. Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) } -func newConnector(ctx context.Context, source *Source) (Connector, error) { - apiEndpoint := source.conn.Endpoint - if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) { - apiEndpoint = cloudV3Endpoint +// isGHECloud returns true if the endpoint is a GHE.com instance +// (GHEC with data residency). GHE.com uses the same root-level API layout as +// api.github.com, NOT the GHES /api/v3/ layout. +// +// Examples: +// +// https://api.company.ghe.com -> true +// https://company.ghe.com -> true (web URL; user might pass this) +// https://github.mycompany.com -> false (GHES) +// https://api.github.com -> false (regular github.com) +func isGHECloud(endpoint string) bool { + u, err := url.Parse(strings.TrimRight(endpoint, "/")) + if err != nil { + return false + } + return strings.HasSuffix(strings.ToLower(u.Hostname()), gheCloudSuffix) +} + +// normalizeGHECloudAPIEndpoint ensures the endpoint points to the API subdomain +// with a trailing slash (required by go-github's BaseURL). +// +// On GHE.com the web UI lives at SUBDOMAIN.ghe.com while the API lives at +// api.SUBDOMAIN.ghe.com. Users may pass either form. +// +// https://company.ghe.com -> https://api.company.ghe.com/ +// https://company.ghe.com/ -> https://api.company.ghe.com/ +// https://api.company.ghe.com -> https://api.company.ghe.com/ +// https://api.company.ghe.com/ -> https://api.company.ghe.com/ +func normalizeGHECloudAPIEndpoint(endpoint string) (string, error) { + u, err := url.Parse(strings.TrimRight(endpoint, "/")) + if err != nil { + return "", fmt.Errorf("invalid GHE.com endpoint URL: %w", err) + } + + host := u.Hostname() + port := u.Port() + + // If user passed the web URL (company.ghe.com), prepend "api." + if !strings.HasPrefix(strings.ToLower(host), "api.") { + host = "api." + host + } + + if port != "" { + u.Host = host + ":" + port + } else { + u.Host = host + } + + // go-github requires a trailing slash on BaseURL. + u.Path = "/" + + return u.String(), nil +} + +// createGHECloudClient creates a go-github client configured for GHE.com. +// GHE.com serves its REST API at the root of api.SUBDOMAIN.ghe.com (same +// layout as api.github.com), so we must NOT use WithEnterpriseURLs which +// would append /api/v3/. +func createGHECloudClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { + apiURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint) + if err != nil { + return nil, err + } + + parsedURL, err := url.Parse(apiURL) + if err != nil { + return nil, fmt.Errorf("failed to parse GHE.com API URL: %w", err) } + client := github.NewClient(httpClient) + client.BaseURL = parsedURL + client.UploadURL = parsedURL + return client, nil +} + +func newConnector(ctx context.Context, source *Source) (Connector, error) { + apiEndpoint := canonicalAPIEndpoint(source.conn.Endpoint) + switch cred := source.conn.GetCredential().(type) { case *sourcespb.GitHub_GithubApp: log.RedactGlobally(cred.GithubApp.GetPrivateKey()) - return NewAppConnector(ctx, apiEndpoint, cred.GithubApp) + return NewAppConnector(apiEndpoint, cred.GithubApp, source.conn.GetScanAllInstallations()) case *sourcespb.GitHub_BasicAuth: log.RedactGlobally(cred.BasicAuth.GetPassword()) return NewBasicAuthConnector(ctx, apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth) @@ -59,27 +137,44 @@ func createAPIClient(ctx context.Context, httpClient *http.Client, apiEndpoint s ctx.Logger().V(2).Info("Creating API client", "url", apiEndpoint) // If we're using public GitHub, make a regular client. - // Otherwise, make an enterprise client. if strings.EqualFold(apiEndpoint, cloudV3Endpoint) { return github.NewClient(httpClient), nil } + // GHE.com (GHEC with data residency) serves its API at the root level + // (like api.github.com), NOT under /api/v3/. + if isGHECloud(apiEndpoint) { + return createGHECloudClient(httpClient, apiEndpoint) + } + + // GHES (self-hosted) — WithEnterpriseURLs appends /api/v3/ and /api/uploads/. return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) } func createGraphqlClient(ctx context.Context, client *http.Client, apiEndpoint string) (*githubv4.Client, error) { var graphqlEndpoint string - if apiEndpoint == cloudV3Endpoint { + switch { + case apiEndpoint == cloudV3Endpoint: graphqlEndpoint = cloudGraphqlEndpoint - } else { - // Use the root endpoint for the host. + + case isGHECloud(apiEndpoint): + // GHE.com: GraphQL lives at the root level, same as api.github.com. + // https://api.SUBDOMAIN.ghe.com/graphql + apiURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint) + if err != nil { + return nil, fmt.Errorf("error normalizing GHE.com endpoint: %w", err) + } + graphqlEndpoint = strings.TrimRight(apiURL, "/") + "/graphql" + + default: + // GHES: GraphQL lives under /api/graphql. // https://docs.github.com/en/enterprise-server@3.11/graphql/guides/introduction-to-graphql parsedURL, err := url.Parse(apiEndpoint) if err != nil { return nil, fmt.Errorf("error parsing URL: %w", err) } - // GitHub Enterprise uses `/api/v3` for the base. (https://github.com/google/go-github/issues/958) + // GitHub Enterprise Server uses `/api/v3` for the base. (https://github.com/google/go-github/issues/958) // Swap it, and anything before `/api`, with GraphQL. before, _ := strings.CutSuffix(parsedURL.Path, "/api/v3") parsedURL.Path = before + "/api/graphql" diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index cbe623e4a429..0739d4c9ae7f 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -2,7 +2,12 @@ package github import ( "fmt" + "net/http" + "net/url" "strconv" + "strings" + "sync" + "time" "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" @@ -16,94 +21,423 @@ import ( ) type appConnector struct { - apiClient *github.Client - graphqlClient *githubv4.Client installationClient *github.Client installationID int64 + appID int64 + appPrivateKey []byte + apiEndpoint string + + mu sync.RWMutex + // clientsByInstallationID is scoped to this connector/source run and is + // bounded by the installations touched while scanning. + clientsByInstallationID map[int64]*appInstallationClients + + // repoInstallationMap maps repo clone URLs to their owning installation + // ID for repos discovered from non-default installations. Clone checks + // this map to use the correct installation token. + repoInstallationMap map[string]int64 +} + +type appInstallationClients struct { + apiClient *github.Client + graphqlClient *githubv4.Client + tokenMu sync.RWMutex + token *github.InstallationToken +} + +func (cs *appInstallationClients) accessTokenIfValid() *github.InstallationToken { + if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) { + return nil + } + return cs.token } var _ Connector = (*appConnector)(nil) -func NewAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (Connector, error) { - installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) - if err != nil { - return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err) +const githubHTTPTimeoutSeconds = 60 + +func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllInstallations bool) (Connector, error) { + var installationID int64 + var err error + + if app.InstallationId != "" { + installationID, err = strconv.ParseInt(app.InstallationId, 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err) + } + } + + if installationID == 0 && !scanAllInstallations { + return nil, fmt.Errorf("githubApp.installationId is required unless scanAllInstallations is set") } appID, err := strconv.ParseInt(app.AppId, 10, 64) if err != nil { - return nil, fmt.Errorf("could not parse app ID %q: %w", appID, err) + return nil, fmt.Errorf("could not parse app ID %q: %w", app.AppId, err) } - const httpTimeoutSeconds = 60 - httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) - - installationTransport, err := ghinstallation.NewAppsTransport( - httpClient.Transport, - appID, - []byte(app.PrivateKey)) + installationTransport, err := newAppsTransport(apiEndpoint, appID, []byte(app.PrivateKey)) if err != nil { return nil, fmt.Errorf("could not create installation client transport: %w", err) } - installationTransport.BaseURL = apiEndpoint - - installationHttpClient := common.RetryableHTTPClientTimeout(60) - installationHttpClient.Transport = installationTransport - installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + installationClient, err := newGitHubClientWithTransport(apiEndpoint, installationTransport) if err != nil { return nil, fmt.Errorf("could not create installation client: %w", err) } - apiTransport, err := ghinstallation.New( - httpClient.Transport, - appID, - installationID, - []byte(app.PrivateKey)) - if err != nil { - return nil, fmt.Errorf("could not create API client transport: %w", err) + connector := &appConnector{ + installationClient: installationClient, + installationID: installationID, + appID: appID, + appPrivateKey: []byte(app.PrivateKey), + apiEndpoint: apiEndpoint, + clientsByInstallationID: make(map[int64]*appInstallationClients), + repoInstallationMap: make(map[string]int64), } - apiTransport.BaseURL = apiEndpoint - httpClient.Transport = apiTransport - apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return nil, fmt.Errorf("could not create API client: %w", err) + // When no default installation is configured (installationID == 0, only + // possible with scanAllInstallations), installations are discovered and + // clients created lazily per-repo, so there is no default client to + // validate up front. + if connector.HasDefaultInstallation() { + _, err := connector.clientsForInstallation(installationID) + if err != nil { + return nil, fmt.Errorf("could not create clients for configured installation: %w", err) + } } - graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint) + return connector, nil +} + +// HasDefaultInstallation reports whether a default installation is +// configured. False only when scanAllInstallations is set and +// githubApp.installationId was omitted, in which case there is no +// authoritative installation to fall back to for repos/gists discovered +// outside any installation's own listing. +func (c *appConnector) HasDefaultInstallation() bool { + return c.installationID != 0 +} + +// APIClient returns the client for the connector's default installation. +// When scanAllInstallations is configured without a githubApp.installationId +// (installationID == 0), NewAppConnector does not pre-create this client, so +// it is created lazily here rather than returning nil: several callers +// (Validate, mapRemainingAccessibleRepos, member gist/repo lookups) use this +// as a fallback client for repos/resources outside any installation listing +// and call it unconditionally. +func (c *appConnector) APIClient() *github.Client { + client, err := c.APIClientForInstallation(c.installationID) if err != nil { - return nil, fmt.Errorf("error creating GraphQL client: %w", err) + return nil } + return client +} - return &appConnector{ - apiClient: apiClient, - graphqlClient: graphqlClient, - installationClient: installationClient, - installationID: installationID, - }, nil +func (c *appConnector) APIClientForRepo(repoURL string) (*github.Client, error) { + installID, _ := c.installationIDForRepo(repoURL) + return c.APIClientForInstallation(installID) } -func (c *appConnector) APIClient() *github.Client { - return c.apiClient +func (c *appConnector) GraphQLClientForRepo(ctx context.Context, repoURL string) (*githubv4.Client, error) { + installID, _ := c.installationIDForRepo(repoURL) + return c.graphqlClientForInstallation(installID) +} + +func (c *appConnector) graphqlClientForInstallation(installID int64) (*githubv4.Client, error) { + clients, err := c.clientsForInstallation(installID) + if err != nil { + return nil, err + } + + return clients.graphqlClient, nil } func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) { - // TODO: Check rate limit for this call. - token, _, err := c.installationClient.Apps.CreateInstallationToken( + installID, _ := c.installationIDForRepo(repoURL) + if installID == 0 { + // installID falls back to the connector's default installationID + // (see installationIDForRepo/ensureRepoInstallation) for repos + // discovered outside any installation's repo listing, e.g. org + // members' personal repos with scanUsers. That default is unset (0) + // when scanAllInstallations is true and githubApp.installationId + // was omitted, since no single installation is authoritative. + return "", nil, fmt.Errorf("no GitHub App installation resolved for repo %q; set githubApp.installationId to a fallback installation to scan repos outside installation listings (e.g. member repos with scanUsers) together with scanAllInstallations", repoURL) + } + + token, err := c.accessTokenForInstallation(ctx, installID) + if err != nil { + return "", nil, err + } + + return git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...) +} + +// accessTokenForInstallation returns an installation access token. +func (c *appConnector) accessTokenForInstallation(ctx context.Context, installID int64) (string, error) { + if installID == 0 { + return "", fmt.Errorf("tried to fetch installation access token for id 0") + } + + clients, err := c.clientsForInstallation(installID) + if err != nil { + return "", err + } + + clients.tokenMu.RLock() + token := clients.accessTokenIfValid() + clients.tokenMu.RUnlock() + + if token != nil { + return token.GetToken(), nil + } + + clients.tokenMu.Lock() + defer clients.tokenMu.Unlock() + + if token := clients.accessTokenIfValid(); token != nil { + return token.GetToken(), nil + } + + token, _, err = c.installationClient.Apps.CreateInstallationToken( ctx, - c.installationID, - &github.InstallationTokenOptions{}) + installID, + &github.InstallationTokenOptions{}, + ) if err != nil { - return "", nil, fmt.Errorf("could not create installation token: %w", err) + return "", fmt.Errorf("could not create installation token for installation %d: %w", installID, err) + } + + if exp := token.GetExpiresAt(); !exp.IsZero() { + // Refresh just before the real expiry + token.ExpiresAt = &github.Timestamp{Time: exp.Add(-time.Minute)} + } + + clients.token = token + + return token.GetToken(), nil +} + +// installationIDForRepo returns the mapped installation ID for repoURL. When no +// mapping exists, it falls back to the configured installation and returns false. +// +// Note: Because ensureRepoInstallation also stores values in this map with +// c.installationID, the second return value here is NOT indicative of using +// the fallback. +func (c *appConnector) installationIDForRepo(repoURL string) (int64, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + if id, ok := c.repoInstallationMap[repoURL]; ok { + return id, true + } + return c.installationID, false +} + +func (c *appConnector) hasRepoInstallation(repoURL string) bool { + c.mu.RLock() + defer c.mu.RUnlock() + + _, ok := c.repoInstallationMap[repoURL] + return ok +} + +// setRepoInstallation records which installation owns a repo and its derived +// wiki URL so Clone uses the correct installation token for cross-org repos. +func (c *appConnector) setRepoInstallation(repoURL string, installationID int64) { + c.setRepoInstallationForWiki(repoURL, installationID, wikiCloneURLForRepo) +} + +// ensureRepoInstallation maps a repo to the default installation unless an +// installation already claims it. Used for repos discovered outside +// installation listings (e.g. org members' personal repos), which no +// installation owns but which the default installation token can still +// access when they are public. The check and set happen under one lock so a +// concurrent mapping to a real installation is never overwritten. +func (c *appConnector) ensureRepoInstallation(repoURL, repoName string) { + c.mu.Lock() + defer c.mu.Unlock() + + if _, ok := c.repoInstallationMap[repoURL]; ok { + return } + c.repoInstallationMap[repoURL] = c.installationID + if wikiURL, ok := wikiCloneURLForRepoName(repoURL, repoName); ok { + if _, taken := c.repoInstallationMap[wikiURL]; !taken { + c.repoInstallationMap[wikiURL] = c.installationID + } + } +} + +func (c *appConnector) setRepoInstallationForRepoName(repoURL, repoName string, installationID int64) { + c.setRepoInstallationForWiki(repoURL, installationID, func(repoURL string) (string, bool) { + return wikiCloneURLForRepoName(repoURL, repoName) + }) +} - return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "", "x-access-token", true, args...) +func (c *appConnector) setRepoInstallationForWiki(repoURL string, installationID int64, wikiURLForRepo func(string) (string, bool)) { + c.mu.Lock() + defer c.mu.Unlock() + + c.repoInstallationMap[repoURL] = installationID + if wikiURL, ok := wikiURLForRepo(repoURL); ok { + c.repoInstallationMap[wikiURL] = installationID + } } +// GraphQLClient returns the GraphQL client for the connector's default +// installation, lazily creating it if needed. See APIClient for why this +// cannot simply return nil when no default installation is configured. func (c *appConnector) GraphQLClient() *githubv4.Client { - return c.graphqlClient + client, err := c.graphqlClientForInstallation(c.installationID) + if err != nil { + return nil + } + return client } func (c *appConnector) InstallationClient() *github.Client { return c.installationClient } + +// APIClientForInstallation creates a GitHub API client scoped to a specific +// installation. This is needed when a GitHub App is installed across multiple +// orgs — each org's API calls must use that org's installation token to get +// proper IP allowlist bypass and permission scoping. +func (c *appConnector) APIClientForInstallation(installationID int64) (*github.Client, error) { + clients, err := c.clientsForInstallation(installationID) + if err != nil { + return nil, err + } + + return clients.apiClient, nil +} + +func (c *appConnector) clientsForInstallation(installationID int64) (*appInstallationClients, error) { + c.mu.RLock() + clients, ok := c.clientsByInstallationID[installationID] + c.mu.RUnlock() + if ok { + return clients, nil + } + + clients, err := c.clientsForInstallationLocked(installationID) + if err != nil { + return nil, err + } + + return clients, nil +} + +func (c *appConnector) clientsForInstallationLocked(installationID int64) (*appInstallationClients, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if clients, ok := c.clientsByInstallationID[installationID]; ok { + return clients, nil + } + + clients, err := c.createAPIClientsForInstallation(installationID) + if err != nil { + return nil, err + } + + c.clientsByInstallationID[installationID] = clients + + return clients, nil +} + +// createAPIClientsForInstallation creates both types of clients at once and +// saves along with the transport in an appInstallationClients, so those +// structs are never in a partially-filled state. +func (c *appConnector) createAPIClientsForInstallation(installationID int64) (*appInstallationClients, error) { + appsTransport, err := newAppsTransport(c.apiEndpoint, c.appID, c.appPrivateKey) + if err != nil { + return nil, fmt.Errorf("could not create app transport for installation %d: %w", installationID, err) + } + + // NewFromAppsTransport inherits BaseURL from appsTransport, which + // newAppsTransport already set correctly (incl. GHE.com). Don't override it + // with the raw endpoint here or GHE.com token refresh would 401. + transport := ghinstallation.NewFromAppsTransport(appsTransport, installationID) + + apiClient, err := newGitHubClientWithTransport(c.apiEndpoint, transport) + if err != nil { + return nil, fmt.Errorf("could not create API client for installation %d: %w", installationID, err) + } + + // the only thing this uses context for is a level 2 log line... + gqlClient, err := createGraphqlClient(context.Background(), apiClient.Client(), c.apiEndpoint) + if err != nil { + return nil, err + } + + return &appInstallationClients{ + apiClient: apiClient, + graphqlClient: gqlClient, + }, nil +} + +func newAppsTransport(apiEndpoint string, appID int64, privateKey []byte) (*ghinstallation.AppsTransport, error) { + httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds) + appsTransport, err := ghinstallation.NewAppsTransport(httpClient.Transport, appID, privateKey) + if err != nil { + return nil, err + } + + // ghinstallation uses BaseURL to build the token exchange URL + // {BaseURL}/app/installations/{id}/access_tokens. For GHE.com this must be + // the api.SUBDOMAIN.ghe.com base with no /api/v3 and no trailing slash + // (which ghinstallation does not expect), otherwise token refresh 401s. + baseURL, err := appsBaseURL(apiEndpoint) + if err != nil { + return nil, err + } + appsTransport.BaseURL = strings.TrimRight(baseURL, "/") + return appsTransport, nil +} + +// appsBaseURL returns the BaseURL that ghinstallation transports should use +// for token exchange/refresh. For GHE.com it resolves to the api.* subdomain; +// for what is likely GHES, it ensures /api/v3 is used; and for anything else +// including github.com it is the endpoint as-is. +func appsBaseURL(apiEndpoint string) (string, error) { + if isGHECloud(apiEndpoint) { + normalizedURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint) + if err != nil { + return "", fmt.Errorf("could not normalize GHE.com endpoint: %w", err) + } + return normalizedURL, nil + } + + u, err := url.Parse(apiEndpoint) + if err != nil { + return "", fmt.Errorf("malformed endpoint url: %w", err) + } + h := u.Hostname() + + if !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + + // If we're not dealing with github.com or GHE, then maybe GHES, which + // needs /api/v3 for ghinstallation find the right place + if !strings.HasPrefix(h, "api.") && !strings.Contains(h, ".api.") { + if !strings.HasSuffix(u.Path, "/api/v3/") { + u.Path += "api/v3/" + } + } + + return u.String(), nil +} + +func newGitHubClientWithTransport(apiEndpoint string, transport http.RoundTripper) (*github.Client, error) { + httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds) + httpClient.Transport = transport + + // GHE.com (GHEC with data residency) serves its REST API at the root of + // api.SUBDOMAIN.ghe.com (like api.github.com), NOT under the GHES /api/v3 path. + if isGHECloud(apiEndpoint) { + return createGHECloudClient(httpClient, apiEndpoint) + } + return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) +} diff --git a/pkg/sources/github/connector_app_test.go b/pkg/sources/github/connector_app_test.go new file mode 100644 index 000000000000..2d5898395bb2 --- /dev/null +++ b/pkg/sources/github/connector_app_test.go @@ -0,0 +1,626 @@ +package github + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-github/v67/github" + "github.com/shurcooL/githubv4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" +) + +func generateTestPrivateKey(t *testing.T) []byte { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) +} + +func TestAPIClientForInstallation(t *testing.T) { + privKey := generateTestPrivateKey(t) + + var mu sync.Mutex + var tokenRequestPaths []string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") { + mu.Lock() + tokenRequestPaths = append(tokenRequestPaths, r.URL.Path) + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "token": "test-token", + "expires_at": "2099-01-01T00:00:00Z", + }) + return + } + + // Default: return empty JSON array for any list endpoint. + _ = json.NewEncoder(w).Encode([]interface{}{}) + })) + defer server.Close() + + connector := &appConnector{ + appID: 12345, + appPrivateKey: privKey, + apiEndpoint: server.URL, + clientsByInstallationID: make(map[int64]*appInstallationClients), + } + + t.Run("creates distinct clients for different installations", func(t *testing.T) { + client1, err := connector.APIClientForInstallation(111) + require.NoError(t, err) + assert.NotNil(t, client1) + + client2, err := connector.APIClientForInstallation(222) + require.NoError(t, err) + assert.NotNil(t, client2) + + assert.NotSame(t, client1, client2, "should be different client instances") + }) + + t.Run("reuses clients for the same installation", func(t *testing.T) { + client1, err := connector.APIClientForInstallation(333) + require.NoError(t, err) + client2, err := connector.APIClientForInstallation(333) + require.NoError(t, err) + + assert.Same(t, client1, client2) + }) + + t.Run("returned client uses the correct installation ID", func(t *testing.T) { + mu.Lock() + tokenRequestPaths = nil + mu.Unlock() + + client, err := connector.APIClientForInstallation(42) + require.NoError(t, err) + + ctx := trContext.Background() + _, _, _ = client.Organizations.ListMembers(ctx, "test-org", nil) + + mu.Lock() + defer mu.Unlock() + require.Len(t, tokenRequestPaths, 1) + assert.Contains(t, tokenRequestPaths[0], "/app/installations/42/access_tokens") + }) + + t.Run("different installations use different tokens", func(t *testing.T) { + mu.Lock() + tokenRequestPaths = nil + mu.Unlock() + + ctx := trContext.Background() + + client1, err := connector.APIClientForInstallation(100) + require.NoError(t, err) + _, _, _ = client1.Organizations.ListMembers(ctx, "org-a", nil) + + client2, err := connector.APIClientForInstallation(200) + require.NoError(t, err) + _, _, _ = client2.Organizations.ListMembers(ctx, "org-b", nil) + + mu.Lock() + defer mu.Unlock() + require.Len(t, tokenRequestPaths, 2) + assert.Contains(t, tokenRequestPaths[0], "/installations/100/") + assert.Contains(t, tokenRequestPaths[1], "/installations/200/") + }) +} + +func TestNewAppConnectorDefaultAPIClientUsesConfiguredInstallation(t *testing.T) { + privKey := generateTestPrivateKey(t) + + var mu sync.Mutex + var tokenRequestPaths []string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") { + mu.Lock() + tokenRequestPaths = append(tokenRequestPaths, r.URL.Path) + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "token": "configured-installation-token", + "expires_at": "2099-01-01T00:00:00Z", + }) + return + } + + _ = json.NewEncoder(w).Encode([]map[string]string{{"login": "alice"}}) + })) + defer server.Close() + + connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{ + PrivateKey: string(privKey), + InstallationId: "4242", + AppId: "12345", + }, false) + require.NoError(t, err) + require.NotNil(t, connector.APIClient()) + require.NotNil(t, connector.GraphQLClient()) + + gotAPIClient, err := connector.APIClientForRepo("https://github.com/default-org/repo.git") + require.NoError(t, err) + assert.Same(t, connector.APIClient(), gotAPIClient) + + gotGraphQLClient, err := connector.GraphQLClientForRepo(trContext.Background(), "https://github.com/default-org/repo.git") + require.NoError(t, err) + assert.Same(t, connector.GraphQLClient(), gotGraphQLClient) + + _, _, err = connector.APIClient().Organizations.ListMembers(trContext.Background(), "test-org", nil) + require.NoError(t, err) + + mu.Lock() + defer mu.Unlock() + require.Len(t, tokenRequestPaths, 1) + assert.Contains(t, tokenRequestPaths[0], "/app/installations/4242/access_tokens") +} + +func TestNewAppConnectorInstallationIDOptionalWithScanAllInstallations(t *testing.T) { + privKey := generateTestPrivateKey(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]map[string]string{}) + })) + defer server.Close() + + // scanAllInstallations=true with no installationId configured should succeed. + connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{ + PrivateKey: string(privKey), + AppId: "12345", + }, true) + require.NoError(t, err) + require.NotNil(t, connector) + + // Without scanAllInstallations, installationId is still required. + _, err = NewAppConnector(server.URL, &credentialspb.GitHubApp{ + PrivateKey: string(privKey), + AppId: "12345", + }, false) + require.Error(t, err) +} + +// TestAPIClientAndGraphQLClientNonNilWithoutDefaultInstallation guards against +// a nil-pointer panic: APIClient/GraphQLClient must never return nil, since +// callers like Validate, mapRemainingAccessibleRepos, and member gist/repo +// lookups call them unconditionally and dereference the result. Before this +// fix, an appConnector with no default installation (installationID == 0, +// the case when scanAllInstallations is true and installationId is omitted) +// returned nil from a bare map lookup instead of lazily creating a client. +func TestAPIClientAndGraphQLClientNonNilWithoutDefaultInstallation(t *testing.T) { + privKey := generateTestPrivateKey(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]map[string]string{}) + })) + defer server.Close() + + connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{ + PrivateKey: string(privKey), + AppId: "12345", + }, true) + require.NoError(t, err) + + assert.NotNil(t, connector.APIClient()) + assert.NotNil(t, connector.GraphQLClient()) +} + +// TestCloneErrorsWithoutResolvedInstallation covers repos that fall back to +// the connector's default installationID (e.g. member personal repos found +// outside any installation's repo listing, see ensureRepoInstallation). When +// scanAllInstallations is true and no githubApp.installationId is +// configured, that default is unset (0); Clone must fail with an actionable +// error instead of attempting to use installation ID 0. +func TestCloneErrorsWithoutResolvedInstallation(t *testing.T) { + connector := &appConnector{ + installationID: 0, + clientsByInstallationID: make(map[int64]*appInstallationClients), + repoInstallationMap: make(map[string]int64), + } + + _, _, err := connector.Clone(trContext.Background(), "https://github.com/some-member/personal-repo.git") + require.Error(t, err) + assert.Contains(t, err.Error(), "no GitHub App installation resolved") +} + +func TestAccessTokenForInstallation(t *testing.T) { + privKey := generateTestPrivateKey(t) + + accessTokenServer := func(accessTokenHandler func(w http.ResponseWriter, r *http.Request)) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") { + accessTokenHandler(w, r) + return + } + + // Default: return empty JSON array for any list endpoint. + _ = json.NewEncoder(w).Encode([]interface{}{}) + })) + } + + t.Run("invalid", func(t *testing.T) { + _, err := (&appConnector{}).accessTokenForInstallation(trContext.Background(), 0) + require.Error(t, err) + }) + + t.Run("success", func(t *testing.T) { + server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "token": "test-token", + "expires_at": "2099-01-01T00:00:00Z", + }) + }) + defer server.Close() + + c, err := NewAppConnector( + server.URL, + &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"}, + false, + ) + require.NoError(t, err) + ac := c.(*appConnector) + + _, err = ac.accessTokenForInstallation(trContext.Background(), 100) + require.NoError(t, err) + }) + + t.Run("rate limit", func(t *testing.T) { + server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-RateLimit-Remaining", "0") + w.WriteHeader(http.StatusForbidden) + + _ = json.NewEncoder(w).Encode(map[string]interface{}{}) + }) + defer server.Close() + + c, err := NewAppConnector( + server.URL, + &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"}, + false, + ) + require.NoError(t, err) + ac := c.(*appConnector) + + _, err = ac.accessTokenForInstallation(trContext.Background(), 100) + require.Error(t, err) + + var gherr *github.RateLimitError + require.ErrorAs(t, err, &gherr) + }) + + t.Run("not found / no installation", func(t *testing.T) { + server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + + _ = json.NewEncoder(w).Encode(map[string]interface{}{}) + }) + defer server.Close() + + c, err := NewAppConnector( + server.URL, + &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"}, + false, + ) + require.NoError(t, err) + ac := c.(*appConnector) + + _, err = ac.accessTokenForInstallation(trContext.Background(), 100) + require.Error(t, err) + }) + + t.Run("reuse", func(t *testing.T) { + var hits atomic.Int64 + + server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "token": "test-token", + "expires_at": "2099-01-01T00:00:00Z", + }) + }) + defer server.Close() + + c, err := NewAppConnector( + server.URL, + &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"}, + false, + ) + require.NoError(t, err) + ac := c.(*appConnector) + + _, err = ac.accessTokenForInstallation(trContext.Background(), 100) + require.NoError(t, err) + + _, err = ac.accessTokenForInstallation(trContext.Background(), 100) + require.NoError(t, err) + + require.Equal(t, int64(1), hits.Load()) + }) + + t.Run("expire", func(t *testing.T) { + var hits atomic.Int64 + server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "token": "test-token", + "expires_at": "2099-01-01T00:00:00Z", + }) + }) + defer server.Close() + + c, err := NewAppConnector( + server.URL, + &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"}, + false, + ) + require.NoError(t, err) + ac := c.(*appConnector) + + _, err = ac.accessTokenForInstallation(trContext.Background(), 100) + require.NoError(t, err) + + cs, err := ac.clientsForInstallation(100) + require.NoError(t, err) + token := cs.accessTokenIfValid() + require.NotNil(t, token) + + token.ExpiresAt = &github.Timestamp{Time: time.Now()} + + token = cs.accessTokenIfValid() + require.Nil(t, token) + + // race these for good measure + var wg sync.WaitGroup + for range 3 { + wg.Go(func() { + _, err := ac.accessTokenForInstallation(trContext.Background(), 100) + require.NoError(t, err) + }) + } + wg.Wait() + + token = cs.accessTokenIfValid() + require.NotNil(t, token) + + require.Equal(t, int64(2), hits.Load()) + }) +} + +func TestAPIClientForInstallationUsesConfiguredClient(t *testing.T) { + defaultClient := github.NewClient(nil) + connector := &appConnector{ + installationID: 100, + clientsByInstallationID: map[int64]*appInstallationClients{ + 100: {apiClient: defaultClient}, + }, + } + + got, err := connector.APIClientForInstallation(100) + require.NoError(t, err) + assert.Same(t, defaultClient, got) +} + +func TestCloneUsesRepoInstallationMap(t *testing.T) { + connector := &appConnector{ + installationID: 100, + repoInstallationMap: make(map[string]int64), + } + + t.Run("uses default installation when no mapping exists", func(t *testing.T) { + installationID, mapped := connector.installationIDForRepo("https://github.com/default-org/repo.git") + assert.False(t, mapped) + assert.Equal(t, int64(100), installationID) + }) + + t.Run("uses mapped installation for cross-org repos", func(t *testing.T) { + connector.setRepoInstallation("https://github.com/other-org/repo.git", 999) + installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") + assert.True(t, mapped) + assert.Equal(t, int64(999), installationID) + }) + + t.Run("uses mapped installation for derived wiki URLs", func(t *testing.T) { + connector.setRepoInstallation("https://github.com/wiki-org/repo.git", 888) + installationID, mapped := connector.installationIDForRepo("https://github.com/wiki-org/repo.wiki.git") + assert.True(t, mapped) + assert.Equal(t, int64(888), installationID) + }) +} + +func TestAPIClientForRepoUsesRepoInstallationMap(t *testing.T) { + privKey := generateTestPrivateKey(t) + + var mu sync.Mutex + var tokenRequestPaths []string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == http.MethodPost && strings.Contains(r.URL.Path, "access_tokens") { + mu.Lock() + tokenRequestPaths = append(tokenRequestPaths, r.URL.Path) + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "token": "test-token", + "expires_at": "2099-01-01T00:00:00Z", + }) + return + } + + _ = json.NewEncoder(w).Encode([]map[string]string{{"login": "alice"}}) + })) + defer server.Close() + + connector := &appConnector{ + installationID: 100, + appID: 12345, + appPrivateKey: privKey, + apiEndpoint: server.URL, + clientsByInstallationID: make(map[int64]*appInstallationClients), + repoInstallationMap: make(map[string]int64), + } + connector.setRepoInstallation("https://github.com/other-org/repo.git", 999) + + client, err := connector.APIClientForRepo("https://github.com/other-org/repo.git") + require.NoError(t, err) + cachedClient, err := connector.APIClientForRepo("https://github.com/other-org/repo.git") + require.NoError(t, err) + assert.Same(t, client, cachedClient) + + _, _, _ = client.Organizations.ListMembers(trContext.Background(), "test-org", nil) + + mu.Lock() + defer mu.Unlock() + require.Len(t, tokenRequestPaths, 1) + assert.Contains(t, tokenRequestPaths[0], "/app/installations/999/access_tokens") +} + +func TestGraphQLClientForRepoCachesClients(t *testing.T) { + privKey := generateTestPrivateKey(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{}) + })) + defer server.Close() + + defaultGraphQLClient := &githubv4.Client{} + connector := &appConnector{ + installationID: 100, + appID: 12345, + appPrivateKey: privKey, + apiEndpoint: server.URL, + clientsByInstallationID: map[int64]*appInstallationClients{ + 100: {graphqlClient: defaultGraphQLClient}, + }, + repoInstallationMap: make(map[string]int64), + } + connector.setRepoInstallation("https://github.com/other-org/repo.git", 999) + + ctx := trContext.Background() + + t.Run("returns default client for default installation repos", func(t *testing.T) { + got, err := connector.GraphQLClientForRepo(ctx, "https://github.com/default-org/repo.git") + require.NoError(t, err) + assert.Same(t, defaultGraphQLClient, got) + }) + + t.Run("caches clients for non-default installations", func(t *testing.T) { + client1, err := connector.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git") + require.NoError(t, err) + assert.NotSame(t, defaultGraphQLClient, client1) + + client2, err := connector.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git") + require.NoError(t, err) + assert.Same(t, client1, client2) + + apiClient, err := connector.APIClientForRepo("https://github.com/other-org/repo.git") + require.NoError(t, err) + + connector.mu.RLock() + cachedClients := connector.clientsByInstallationID[999] + connector.mu.RUnlock() + require.NotNil(t, cachedClients) + assert.Same(t, apiClient, cachedClients.apiClient) + assert.Same(t, client1, cachedClients.graphqlClient) + }) +} + +func TestDefaultConnectorsReturnDefaultClientsForRepo(t *testing.T) { + ctx := trContext.Background() + apiClient := github.NewClient(nil) + graphqlClient := &githubv4.Client{} + + connectors := []Connector{ + &basicAuthConnector{apiClient: apiClient, graphqlClient: graphqlClient}, + &tokenConnector{apiClient: apiClient, graphqlClient: graphqlClient}, + &unauthenticatedConnector{apiClient: apiClient, graphqlClient: graphqlClient}, + } + + for _, connector := range connectors { + gotAPIClient, err := connector.APIClientForRepo("https://github.com/trufflesecurity/trufflehog.git") + require.NoError(t, err) + assert.Same(t, apiClient, gotAPIClient) + + gotGraphQLClient, err := connector.GraphQLClientForRepo(ctx, "https://github.com/trufflesecurity/trufflehog.git") + require.NoError(t, err) + assert.Same(t, graphqlClient, gotGraphQLClient) + } +} + +func TestAddMembersByOrgWithClient(t *testing.T) { + strPtr := func(s string) *string { return &s } + memberPage := []*github.User{ + {Login: strPtr("alice")}, + {Login: strPtr("bob")}, + {Login: strPtr("charlie")}, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(memberPage) + })) + defer server.Close() + + client, err := github.NewClient(nil).WithEnterpriseURLs(server.URL, server.URL) + require.NoError(t, err) + + s := &Source{ + memberCache: make(map[string]struct{}), + } + + ctx := trContext.Background() + err = s.addMembersByOrgWithClient(ctx, client, "test-org", nil) + require.NoError(t, err) + + assert.Len(t, s.memberCache, 3) + assert.Contains(t, s.memberCache, "alice") + assert.Contains(t, s.memberCache, "bob") + assert.Contains(t, s.memberCache, "charlie") +} + +func TestAppsBaseURL(t *testing.T) { + u, err := appsBaseURL("https://foo.ghe.com/") + assert.NoError(t, err) + assert.Equal(t, "https://api.foo.ghe.com/", u) + + u, err = appsBaseURL("https://ghes.intranet/") + assert.NoError(t, err) + assert.Equal(t, "https://ghes.intranet/api/v3/", u) + + u, err = appsBaseURL("https://ghes.intranet/api/v3/") + assert.NoError(t, err) + assert.Equal(t, "https://ghes.intranet/api/v3/", u) + + u, err = appsBaseURL("https://api.github.com") + assert.NoError(t, err) + assert.Equal(t, "https://api.github.com/", u) + + u, err = appsBaseURL("https://api.github.com/") + assert.NoError(t, err) + assert.Equal(t, "https://api.github.com/", u) +} diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 94aae56abeaf..1b5e218a74e5 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -54,6 +54,10 @@ func (c *basicAuthConnector) APIClient() *github.Client { return c.apiClient } +func (c *basicAuthConnector) APIClientForRepo(_ string) (*github.Client, error) { + return c.apiClient, nil +} + func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) { return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.clonePath, c.username, true, args...) } @@ -61,3 +65,7 @@ func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string, args ... func (c *basicAuthConnector) GraphQLClient() *githubv4.Client { return c.graphqlClient } + +func (c *basicAuthConnector) GraphQLClientForRepo(_ context.Context, _ string) (*githubv4.Client, error) { + return c.graphqlClient, nil +} diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 9bba8c1242f8..1e4d234177ca 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -49,11 +49,16 @@ func NewTokenConnector(ctx context.Context, apiEndpoint, token, clonePath string return nil, fmt.Errorf("error creating GraphQL client: %w", err) } + // Treat both GHES and GHE.com as "enterprise" for enumeration purposes. + // GHE.com (GHEC with data residency) is a dedicated enterprise environment + // with EMU, so enterprise-level enumeration (e.g., addAllVisibleOrgs) applies. + isEnterprise := !strings.EqualFold(apiEndpoint, cloudV3Endpoint) + return &tokenConnector{ apiClient: apiClient, graphqlClient: graphqlClient, token: token, - isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudV3Endpoint), + isGitHubEnterprise: isEnterprise, handleRateLimit: handleRateLimit, authInUrl: authInUrl, clonePath: clonePath, @@ -64,6 +69,10 @@ func (c *tokenConnector) APIClient() *github.Client { return c.apiClient } +func (c *tokenConnector) APIClientForRepo(_ string) (*github.Client, error) { + return c.apiClient, nil +} + func (c *tokenConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) { if err := c.setUserIfUnset(ctx); err != nil { return "", nil, err @@ -76,6 +85,10 @@ func (c *tokenConnector) GraphQLClient() *githubv4.Client { return c.graphqlClient } +func (c *tokenConnector) GraphQLClientForRepo(_ context.Context, _ string) (*githubv4.Client, error) { + return c.graphqlClient, nil +} + func (c *tokenConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 8f93baa83eba..792a0084de8f 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -45,6 +45,10 @@ func (c *unauthenticatedConnector) APIClient() *github.Client { return c.apiClient } +func (c *unauthenticatedConnector) APIClientForRepo(_ string) (*github.Client, error) { + return c.apiClient, nil +} + func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) { return git.CloneRepoUsingUnauthenticated(ctx, repoURL, c.clonePath, args...) } @@ -52,3 +56,7 @@ func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string, ar func (c *unauthenticatedConnector) GraphQLClient() *githubv4.Client { return c.graphqlClient } + +func (c *unauthenticatedConnector) GraphQLClientForRepo(_ context.Context, _ string) (*githubv4.Client, error) { + return c.graphqlClient, nil +} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 4106e97e2d08..1ccc72714b4a 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -1,6 +1,7 @@ package github import ( + "encoding/json" "errors" "fmt" "io" @@ -9,7 +10,6 @@ import ( "net/url" "os" "path/filepath" - "regexp" "slices" "sort" "strings" @@ -17,8 +17,10 @@ import ( "sync/atomic" "time" + "github.com/go-git/go-git/v5/plumbing" "github.com/gobwas/glob" "github.com/google/go-github/v67/github" + "github.com/shurcooL/githubv4" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -64,11 +66,14 @@ type Source struct { scanOptMu sync.Mutex // protects the scanOptions scanOptions *git.ScanOptions - conn *sourcespb.GitHub - jobPool *errgroup.Group - resumeInfoMutex sync.Mutex - resumeInfoSlice []string - connector Connector + conn *sourcespb.GitHub + jobPool *errgroup.Group + resumeInfoMutex sync.Mutex + resumeInfoSlice []string + connector Connector + appInstallationsMu sync.Mutex + appInstallations []*github.Installation + appInstallationsLoaded bool includePRComments bool includeIssueComments bool @@ -77,7 +82,6 @@ type Source struct { commentsTimeframeDays uint32 sources.Progress - sources.CommonSourceUnitUnmarshaller useAuthInUrl bool // pass credentials in the repository urls for cloning } @@ -93,6 +97,13 @@ var _ sources.SourceUnit = (*GistUnit)(nil) type RepoUnit struct { Name string `json:"name"` URL string `json:"url"` + // InstallationID is the GitHub App installation that owns this repo, set + // during scan-all-installations enumeration. Carrying it on the unit lets + // ChunkUnit clone with the correct installation token without re-deriving + // the mapping, which otherwise re-lists every installation's repos on every + // scanned unit (see INT-790). Zero when the source is not an installation + // scan or the unit predates this field. + InstallationID int64 `json:"installation_id,omitempty"` } func (r RepoUnit) SourceUnitID() (string, sources.SourceUnitKind) { return r.URL, "repo" } @@ -106,6 +117,66 @@ type GistUnit struct { func (g GistUnit) SourceUnitID() (string, sources.SourceUnitKind) { return g.URL, "gist" } func (g GistUnit) Display() string { return g.Name } +// unitEnvelope is just the JSON equivalent of apipb.SourceUnit, so we can +// unmarshal source unit payloads. +type unitEnvelope struct { + ID string `json:"id"` + Kind sources.SourceUnitKind `json:"kind,omitempty"` + Name string `json:"display,omitempty"` + UnitData []byte `json:"unit_data,omitempty"` +} + +func (u unitEnvelope) SourceUnitID() (string, sources.SourceUnitKind) { return u.ID, u.Kind } +func (u unitEnvelope) Display() string { return u.Name } + +func unmarshalSourceUnit[unitType sources.SourceUnit](data []byte) (unitType, error) { + u := new(unitType) + + if err := json.Unmarshal(data, u); err != nil { + return *new(unitType), err + } + + return *u, nil +} + +func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) { + env, err := unmarshalSourceUnit[unitEnvelope](data) + if err != nil { + return nil, err + } + + switch env.Kind { + case "": // data is a raw SourceUnit, no envelope + // Look for a raw RepoUnit first + ru, err := unmarshalSourceUnit[RepoUnit](data) + if err != nil { + return nil, err + } + + if ru.URL != "" { // This is the test for a valid RepoUnit + return ru, nil + } + + if env.ID == "" { // This is the test for a valid generic GitHub Source Unit + return nil, errors.New("not a github source unit") + } + + return env, nil + case "repo": + if len(env.UnitData) > 0 { + return unmarshalSourceUnit[RepoUnit](env.UnitData) + } + return RepoUnit{Name: env.Name, URL: env.ID}, nil + case "gist": + if len(env.UnitData) > 0 { + return unmarshalSourceUnit[GistUnit](env.UnitData) + } + return GistUnit{Name: env.Name, URL: env.ID}, nil + default: + return nil, fmt.Errorf("unrecognized source unit kind %q", env.Kind) + } +} + // -------------------------------------------------------------------------------- // WithCustomContentWriter sets the useCustomContentWriter flag on the source. @@ -127,8 +198,6 @@ var _ sources.Source = (*Source)(nil) var _ sources.SourceUnitUnmarshaller = (*Source)(nil) var _ sources.SourceUnitEnumChunker = (*Source)(nil) -var endsWithGithub = regexp.MustCompile(`github\.com/?$`) - // Type returns the type of source. // It is used for matching source types in configuration and job input. func (s *Source) Type() sourcespb.SourceType { @@ -239,6 +308,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so // configuration uses the inverse logic of the `useAuthInUrl` flag. s.useAuthInUrl = !s.conn.RemoveAuthInUrl + s.resetAppInstallations() connector, err := newConnector(aCtx, s) if err != nil { @@ -279,6 +349,17 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so return fmt.Errorf("cannot specify head or base with multiple repositories") } + if connector, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations { + if _, err := s.getAppInstallations(aCtx, connector); err != nil { + return err + } + if len(s.repos) > 0 { + if err := s.mapExplicitReposToInstallations(aCtx, connector); err != nil { + return err + } + } + } + cfg := &git.Config{ SourceName: s.name, JobID: s.jobID, @@ -321,6 +402,17 @@ func (s *Source) Validate(ctx context.Context) []error { - Returns 401 for invalid credentials but works with no auth (as unauthenticated) - Doesn't consume API quota when called */ + // When scanAllInstallations is set with no githubApp.installationId, + // there's no default installation token to call RateLimit with (see + // appConnector.HasDefaultInstallation). Validate the App credentials + // directly against the app-level (JWT-authenticated) client instead. + if connector, ok := s.connector.(*appConnector); ok && !connector.HasDefaultInstallation() { + if _, _, err := connector.InstallationClient().Apps.Get(ctx, ""); err != nil { + return []error{err} + } + return nil + } + if _, _, err := s.connector.APIClient().RateLimit.Get(ctx); err != nil { return []error{err} } @@ -331,8 +423,10 @@ func (s *Source) Validate(ctx context.Context) []error { func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metadatapb.Visibility { // It isn't possible to get the visibility of a wiki. // We must use the visibility of the corresponding repository. - if strings.HasSuffix(repoURL, ".wiki.git") { - repoURL = strings.TrimSuffix(repoURL, ".wiki.git") + ".git" + if _, ok := s.repoInfoCache.get(repoURL); !ok { + if parentRepoURL, isWiki := repoCloneURLForWikiCloneURL(repoURL); isWiki { + repoURL = parentRepoURL + } } repoInfo, ok := s.repoInfoCache.get(repoURL) @@ -364,17 +458,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar // We don't care about handling enumerated values as they happen during // the normal Chunks flow because we enumerate and scan in two steps. + var enumerationErrs []error noopReporter := sources.VisitorReporter{ VisitUnit: func(context.Context, sources.SourceUnit) error { return nil }, + VisitErr: func(ctx context.Context, err error) error { + enumerationErrs = append(enumerationErrs, err) + return ctx.Err() + }, } err := s.Enumerate(ctx, noopReporter) if err != nil { - return fmt.Errorf("error enumerating: %w", err) + return errors.Join(fmt.Errorf("error enumerating: %w", err), errors.Join(enumerationErrs...)) } - return s.scan(ctx, chunksReporter) + return errors.Join(append(enumerationErrs, s.scan(ctx, chunksReporter))...) } // Enumerate enumerates the GitHub source based on authentication method and @@ -412,7 +511,16 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e } continue } - if err := dedupeReporter.UnitOk(ctx, RepoUnit{Name: name, URL: url}); err != nil { + + var installationID int64 = 0 + if ac, ok := s.connector.(*appConnector); ok { + ac.ensureRepoInstallation(url, name) + installationID, _ = ac.installationIDForRepo(url) + } + + ru := RepoUnit{Name: name, URL: url, InstallationID: installationID} + + if err := dedupeReporter.UnitOk(ctx, ru); err != nil { return err } } @@ -421,7 +529,7 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e // this felt like a compromise that allowed me to isolate connection logic without rewriting the entire source. switch c := s.connector.(type) { case *appConnector: - if err := s.enumerateWithApp(ctx, c.InstallationClient(), dedupeReporter); err != nil { + if err := s.enumerateWithApp(ctx, c, dedupeReporter); err != nil { return err } case *basicAuthConnector: @@ -487,6 +595,9 @@ func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string, reporter } if !s.ignoreGists && isGistUrl(urlParts) { + if connector, ok := s.connector.(*appConnector); ok && !connector.HasDefaultInstallation() { + return repo, fmt.Errorf("cannot fetch gist %q: githubApp.installationId is required as a fallback installation to scan gists together with scanAllInstallations", repo) + } // Cache gist info. for { gistID := extractGistID(urlParts) @@ -509,7 +620,12 @@ func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string, reporter } else { // Cache repository info. for { - ghRepo, _, err := s.connector.APIClient().Repositories.Get(ctx, urlParts[1], urlParts[2]) + apiClient, err := s.connector.APIClientForRepo(repo) + if err != nil { + return repo, fmt.Errorf("failed to create API client for repository: %w", err) + } + + ghRepo, _, err := apiClient.Repositories.Get(ctx, urlParts[1], urlParts[2]) if s.handleRateLimit(ctx, err, reporter) { continue } @@ -517,6 +633,14 @@ func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string, reporter return repo, fmt.Errorf("failed to fetch repository: %w", err) } s.cacheRepoInfo(ghRepo) + // Handle repository redirects: the API returns the canonical CloneURL + // which may differ from the original URL (e.g. org rename). Cache info + // under the original URL too so scanRepo can look it up. + if cloneURL := ghRepo.GetCloneURL(); cloneURL != repo { + if info, ok := s.repoInfoCache.get(cloneURL); ok { + s.repoInfoCache.put(repo, info) + } + } break } } @@ -623,20 +747,35 @@ func (s *Source) enumerateWithToken(ctx context.Context, isGithubEnterprise bool return nil } -func (s *Source) enumerateWithApp(ctx context.Context, installationClient *github.Client, reporter sources.UnitReporter) error { +func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector, reporter sources.UnitReporter) error { // If no repos were provided, enumerate them. if len(s.repos) == 0 { - if err := s.getReposByApp(ctx, reporter); err != nil { - return err + if s.conn.ScanAllInstallations { + if err := s.enumerateAllInstallationRepos(ctx, connector, reporter); err != nil { + return err + } + } else { + if err := s.getReposByApp(ctx, reporter); err != nil { + return err + } } // Check if we need to find user repos. if s.conn.ScanUsers { - err := s.addMembersByApp(ctx, installationClient, reporter) + err := s.addMembersByApp(ctx, connector, reporter) if err != nil { return err } ctx.Logger().Info("Scanning repos", "org_members", len(s.memberCache)) + if !connector.HasDefaultInstallation() { + // Member repos/gists live outside any installation's own + // listing, so there is no installation token to fetch them + // with (see appConnector.HasDefaultInstallation). Skip with a + // clear message instead of letting each per-member API call + // fail against installation ID 0. + ctx.Logger().Info("Skipping org member repos and gists: githubApp.installationId is required as a fallback installation to scan members together with scanAllInstallations", "org_members", len(s.memberCache)) + return nil + } // TODO: Replace loop below with a call to s.addReposForMembers(ctx, reporter) for member := range s.memberCache { logger := ctx.Logger().WithValues("member", member) @@ -655,6 +794,515 @@ func (s *Source) enumerateWithApp(ctx context.Context, installationClient *githu return nil } +func (s *Source) resetAppInstallations() { + s.appInstallationsMu.Lock() + defer s.appInstallationsMu.Unlock() + + s.appInstallations = nil + s.appInstallationsLoaded = false +} + +func (s *Source) getAppInstallations(ctx context.Context, connector *appConnector) ([]*github.Installation, error) { + s.appInstallationsMu.Lock() + defer s.appInstallationsMu.Unlock() + + if s.appInstallationsLoaded { + return s.appInstallations, nil + } + + installs, err := s.listAppInstallations(ctx, connector, defaultPagination) + if err != nil { + return nil, err + } + s.appInstallations = installs + s.appInstallationsLoaded = true + return s.appInstallations, nil +} + +func (s *Source) listAppInstallations( + ctx context.Context, + connector *appConnector, + perPage int, +) ([]*github.Installation, error) { + installationClient := connector.InstallationClient() + opts := &github.ListOptions{PerPage: perPage} + + var allInstalls []*github.Installation + for { + installs, res, err := installationClient.Apps.ListInstallations(ctx, opts) + if s.handleRateLimit(ctx, err) { + continue + } + if err != nil { + return nil, fmt.Errorf("could not list installations: %w", err) + } + allInstalls = append(allInstalls, installs...) + if res == nil || res.NextPage == 0 { + break + } + opts.Page = res.NextPage + } + + return allInstalls, nil +} + +func (s *Source) listAppInstallationRepoPage( + ctx context.Context, + client *github.Client, + opts *github.ListOptions, + handleRateLimit func(error) bool, +) ([]*github.Repository, *github.Response, error) { + for { + result, res, err := client.Apps.ListRepos(ctx, opts) + if handleRateLimit(err) { + continue + } + if result == nil { + return nil, res, err + } + if result.Repositories == nil { + return []*github.Repository{}, res, err + } + return result.Repositories, res, err + } +} + +// enumerateAllInstallationRepos discovers repos from every installation of the +// GitHub App. For each installation, it creates a per-installation API client +// and records the installation ID for each repo so Clone uses the correct token. +// Per-installation failures are reported through the unit reporter and don't +// abort enumeration of the remaining installations. +func (s *Source) enumerateAllInstallationRepos(ctx context.Context, connector *appConnector, reporter sources.UnitReporter) error { + installs, err := s.getAppInstallations(ctx, connector) + if err != nil { + return err + } + + for _, install := range installs { + installID := install.GetID() + account := install.GetAccount().GetLogin() + installCtx := context.WithValues(ctx, "org", account, "installation_id", installID) + installCtx.Logger().Info("Enumerating repos from GitHub App installation") + + client, err := connector.APIClientForInstallation(installID) + if err != nil { + installCtx.Logger().Error(err, "could not create API client for installation") + if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", installID, err)); err != nil { + return err + } + continue + } + + listRepos := func(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { + repos, resp, err := s.listAppInstallationRepoPage(ctx, client, opts.getListOptions(), func(err error) bool { + return s.handleRateLimitWithUnitReporter(ctx, reporter, err) + }) + for _, r := range repos { + connector.setRepoInstallationForRepoName(r.GetCloneURL(), r.GetName(), installID) + } + return repos, resp, err + } + + if err := s.processRepos(installCtx, account, reporter, listRepos, &appListOptions{ + ListOptions: github.ListOptions{PerPage: defaultPagination}, + }); err != nil { + installCtx.Logger().Error(err, "error enumerating repos for installation") + if err := reporter.UnitErr(installCtx, fmt.Errorf("error enumerating repos for installation %d: %w", installID, err)); err != nil { + return err + } + continue + } + } + + return nil +} + +func (s *Source) mapExplicitReposToInstallations(ctx context.Context, connector *appConnector) error { + return s.mapReposToInstallations(ctx, connector, s.repos) +} + +type repoMappingRequest struct { + original string + normalized string + lookupURLs []string + lookupKeys []string +} + +func (s *Source) mapReposToInstallations(ctx context.Context, connector *appConnector, repos []string) error { + if len(repos) == 0 { + return nil + } + + wantedRepos := make(map[string]repoMappingRequest, len(repos)) + for _, repo := range repos { + normalized, err := s.normalizeRepo(repo) + if err != nil { + return fmt.Errorf("could not normalize configured repo %q: %w", repo, err) + } + if connector.hasRepoInstallation(repo) || connector.hasRepoInstallation(normalized) { + continue + } + request := repoMappingRequest{ + original: repo, + normalized: normalized, + lookupURLs: repoURLsForInstallationLookup(normalized), + } + for _, lookupURL := range request.lookupURLs { + key, err := repoHostOwnerNameKey(lookupURL) + if err != nil { + return fmt.Errorf("could not parse configured repo %q: %w", repo, err) + } + request.lookupKeys = append(request.lookupKeys, key) + } + // During installation repo listing, only allow exact matches. If no + // exact repo exists, mapRemainingReposByMetadata tries wiki-parent + // fallbacks afterward. + wantedRepos[request.lookupKeys[0]] = request + } + if len(wantedRepos) == 0 { + return nil + } + + installs, err := s.getAppInstallations(ctx, connector) + if err != nil { + return err + } + + var errs error + clients := make(map[int64]*github.Client, len(installs)) + installedRepoKeys := make(map[int64]map[string]struct{}, len(installs)) + for _, install := range installs { + installID := install.GetID() + client, err := connector.APIClientForInstallation(installID) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("could not create API client for installation %d: %w", installID, err)) + continue + } + clients[installID] = client + + opts := &github.ListOptions{PerPage: defaultPagination} + for { + repos, res, err := s.listAppInstallationRepoPage(ctx, client, opts, func(err error) bool { + return s.handleRateLimit(ctx, err) + }) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("could not list repos for installation %d: %w", installID, err)) + break + } + + if repos != nil { + if installedRepoKeys[installID] == nil { + installedRepoKeys[installID] = make(map[string]struct{}, len(repos)) + } + for _, repo := range repos { + key, err := repoHostOwnerNameKey(repo.GetCloneURL()) + if err == nil { + installedRepoKeys[installID][key] = struct{}{} + } + } + s.recordInstallationRepoMatches(connector, installID, repos, wantedRepos) + if len(wantedRepos) == 0 { + if errs != nil { + ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos") + } + return nil + } + } + + if res == nil || res.NextPage == 0 { + break + } + opts.Page = res.NextPage + } + } + + if len(wantedRepos) > 0 { + errs = errors.Join(errs, s.mapRemainingReposByMetadata(ctx, connector, installs, clients, installedRepoKeys, wantedRepos)) + if len(wantedRepos) == 0 { + if errs != nil { + ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos") + } + return nil + } + } + + if len(wantedRepos) > 0 { + errs = errors.Join(errs, s.mapRemainingAccessibleRepos(ctx, connector, wantedRepos)) + if len(wantedRepos) == 0 { + // Every requested repo was mapped, so returning errs would fail + // repos that succeeded; log partial enumeration errors instead. + if errs != nil { + ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos") + } + return nil + } + } + + if len(wantedRepos) > 0 { + unmatched := make([]string, 0, len(wantedRepos)) + for _, requested := range wantedRepos { + unmatched = append(unmatched, requested.original) + } + sort.Strings(unmatched) + errs = errors.Join(errs, fmt.Errorf("configured repos were not found in any GitHub App installation: %s", strings.Join(unmatched, ", "))) + } + + return errs +} + +func (s *Source) recordInstallationRepoMatches(connector *appConnector, installationID int64, repos []*github.Repository, wantedRepos map[string]repoMappingRequest) { + for _, repo := range repos { + cloneURL := repo.GetCloneURL() + key, err := repoHostOwnerNameKey(cloneURL) + if err != nil { + continue + } + requested, ok := wantedRepos[key] + if !ok { + continue + } + deleteRepoMappingRequest(wantedRepos, requested) + s.recordRepoInstallation(connector, installationID, repo, requested) + } +} + +func deleteRepoMappingRequest(wantedRepos map[string]repoMappingRequest, requested repoMappingRequest) { + if len(requested.lookupKeys) > 0 { + delete(wantedRepos, requested.lookupKeys[0]) + } +} + +func (s *Source) mapRemainingReposByMetadata( + ctx context.Context, + connector *appConnector, + installs []*github.Installation, + clients map[int64]*github.Client, + installedRepoKeys map[int64]map[string]struct{}, + wantedRepos map[string]repoMappingRequest, +) error { + var errs error + for _, install := range installs { + if len(wantedRepos) == 0 { + return errs + } + + installID := install.GetID() + client, ok := clients[installID] + if !ok { + continue + } + + for _, requested := range wantedRepos { + for i, lookupURL := range requested.lookupURLs { + _, parts, err := getRepoURLParts(lookupURL) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("could not parse configured repo %q: %w", requested.original, err)) + break + } + + var repo *github.Repository + for { + repo, _, err = client.Repositories.Get(ctx, parts[1], parts[2]) + if s.handleRateLimit(ctx, err) { + continue + } + break + } + if err != nil { + if isGitHub404Error(err) { + continue + } + errs = errors.Join(errs, fmt.Errorf("could not fetch repo %q for installation %d: %w", requested.original, installID, err)) + break + } + canonicalKey, err := repoHostOwnerNameKey(repo.GetCloneURL()) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("could not parse fetched repo %q for installation %d: %w", repo.GetCloneURL(), installID, err)) + continue + } + if _, ok := installedRepoKeys[installID][canonicalKey]; !ok { + continue + } + if !sameRepoHost(requested.normalized, repo.GetCloneURL()) { + if i == 0 { + break + } + continue + } + + deleteRepoMappingRequest(wantedRepos, requested) + s.recordRepoInstallation(connector, installID, repo, requested) + if len(wantedRepos) == 0 { + return errs + } + break + } + } + } + return errs +} + +// mapRemainingAccessibleRepos handles repos that no installation owns, such +// as org members' personal repos enumerated via scan_users. Installations +// only ever contain repos owned by the installing account, but the default +// installation token can still read repos that are visible to it (public +// repos in particular), so a repo that the API confirms accessible is mapped +// to the default installation instead of failing the scan. +func (s *Source) mapRemainingAccessibleRepos( + ctx context.Context, + connector *appConnector, + wantedRepos map[string]repoMappingRequest, +) error { + if !connector.HasDefaultInstallation() { + return fmt.Errorf("cannot resolve %d configured repo(s) outside installation listings: githubApp.installationId is required as a fallback installation together with scanAllInstallations", len(wantedRepos)) + } + + var errs error + client := connector.APIClient() + for _, requested := range wantedRepos { + for i, lookupURL := range requested.lookupURLs { + _, parts, err := getRepoURLParts(lookupURL) + if err != nil { + errs = errors.Join(errs, + fmt.Errorf("could not parse configured repo %q: %w", requested.original, err)) + break + } + + var repo *github.Repository + for { + repo, _, err = client.Repositories.Get(ctx, parts[1], parts[2]) + if s.handleRateLimit(ctx, err) { + continue + } + break + } + if err != nil { + if isGitHub404Error(err) { + continue + } + errs = errors.Join(errs, + fmt.Errorf("could not fetch repo %q with default installation: %w", requested.original, err)) + break + } + if !sameRepoHost(requested.normalized, repo.GetCloneURL()) { + if i == 0 { + break + } + continue + } + + ctx.Logger().Info("repo is not owned by any app installation; scanning with the default installation token", + "repo", requested.original) + deleteRepoMappingRequest(wantedRepos, requested) + s.recordRepoInstallation(connector, connector.installationID, repo, requested) + break + } + } + return errs +} + +func (s *Source) recordRepoInstallation(connector *appConnector, installationID int64, repo *github.Repository, requested repoMappingRequest) { + cloneURL := repo.GetCloneURL() + connector.setRepoInstallationForRepoName(cloneURL, repo.GetName(), installationID) + if requested.normalized != cloneURL && sameRepoHost(requested.normalized, cloneURL) { + connector.setRepoInstallation(requested.normalized, installationID) + } + if requested.original != requested.normalized && requested.original != cloneURL && sameRepoHost(requested.original, cloneURL) { + connector.setRepoInstallation(requested.original, installationID) + } + + s.cacheRepoInfo(repo) + if info, ok := s.repoInfoCache.get(cloneURL); ok { + if requested.normalized != cloneURL && sameRepoHost(requested.normalized, cloneURL) { + s.repoInfoCache.put(requested.normalized, info) + } + if requested.original != requested.normalized && requested.original != cloneURL && sameRepoHost(requested.original, cloneURL) { + s.repoInfoCache.put(requested.original, info) + } + } +} + +func repoHostOwnerNameKey(repoURL string) (string, error) { + _, parts, err := getRepoURLParts(repoURL) + if err != nil { + return "", err + } + if len(parts) != 3 { + return "", fmt.Errorf("expected repository URL, got %q", repoURL) + } + return strings.ToLower(parts[0] + "/" + parts[1] + "/" + parts[2]), nil +} + +func sameRepoHost(repoURL, otherRepoURL string) bool { + _, parts, err := getRepoURLParts(repoURL) + if err != nil { + return false + } + _, otherParts, err := getRepoURLParts(otherRepoURL) + if err != nil { + return false + } + return strings.EqualFold(parts[0], otherParts[0]) +} + +func (s *Source) repoURLFromTargetMetadata(meta *source_metadatapb.Github) (string, error) { + linkRepo, err := s.repoURLFromTargetLink(meta.GetLink()) + if err != nil { + return "", err + } + + if repo := meta.GetRepository(); repo != "" { + // Target metadata may only include a bare repo name. In that case the + // owner and host from the link are the authoritative repository identity. + if !strings.Contains(repo, "/") && !strings.Contains(repo, ":") { + return linkRepo, nil + } + normalized, err := s.normalizeRepo(repo) + if err != nil { + return "", fmt.Errorf("could not normalize target repository %q: %w", repo, err) + } + linkKey, err := repoHostOwnerNameKey(linkRepo) + if err != nil { + return "", err + } + repoKey, err := repoHostOwnerNameKey(normalized) + if err != nil { + return "", fmt.Errorf("could not parse target repository %q: %w", repo, err) + } + if repoKey != linkKey && isWikiLink(meta.GetLink()) { + if parentRepoURL, isWiki := repoCloneURLForWikiCloneURL(normalized); isWiki { + repoKey, err = repoHostOwnerNameKey(parentRepoURL) + if err != nil { + return "", fmt.Errorf("could not parse target repository %q: %w", repo, err) + } + } + } + if repoKey != linkKey { + return "", fmt.Errorf("target repository %q does not match link repository %q", normalized, linkRepo) + } + } + + return linkRepo, nil +} + +func (s *Source) repoURLFromTargetLink(link string) (string, error) { + u, err := url.Parse(link) + if err != nil { + return "", fmt.Errorf("unable to parse GitHub URL: %w", err) + } + + segments := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(segments) < 2 { + return "", fmt.Errorf("invalid GitHub URL") + } + + repoURL := (&url.URL{ + Scheme: u.Scheme, + Host: u.Host, + }).JoinPath(segments[0], segments[1]+".git").String() + return s.normalizeRepo(repoURL) +} + func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error { var scannedCount uint64 = 1 @@ -717,20 +1365,22 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources. // Scan the wiki, if enabled, and the repo has one. if s.conn.IncludeWikis && repoInfo.hasWiki && s.wikiIsReachable(ctx, repoURL) { - wikiURL := strings.TrimSuffix(repoURL, ".git") + ".wiki.git" - wikiCtx := context.WithValue(ctx, "repo", wikiURL) + wikiURL, ok := wikiCloneURLForRepoInfo(repoURL, repoInfo) + if ok { + wikiCtx := context.WithValue(ctx, "repo", wikiURL) - _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter) - if err != nil { - // Ignore "Repository not found" errors. - // It's common for GitHub's API to say a repo has a wiki when it doesn't. - if !strings.Contains(err.Error(), "not found") { - if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil { - return err + _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter) + if err != nil { + // Ignore "Repository not found" errors. + // It's common for GitHub's API to say a repo has a wiki when it doesn't. + if !strings.Contains(err.Error(), "not found") { + if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil { + return err + } } - } - // Don't return, it still might be possible to scan comments. + // Don't return, it still might be possible to scan comments. + } } } @@ -761,7 +1411,7 @@ func (s *Source) cloneAndScanRepo(ctx context.Context, repoURL string, repoInfo // if legacy JSON is enabled, don't remove the directory because we need it for outputting legacy JSON. if !s.conn.GetPrintLegacyJson() { if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.conn.NoCleanup && s.conn.GetClonePath() != "") { - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() } } @@ -867,9 +1517,25 @@ func (s *Source) handleRateLimit(ctx context.Context, errIn error, reporters ... } githubNumRateLimitEncountered.WithLabelValues(s.name).Inc() - time.Sleep(retryAfter) - githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds()) - return true + slept, canceled := sleepWithContext(ctx, retryAfter) + githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(slept.Seconds()) + return !canceled +} + +func sleepWithContext(ctx context.Context, duration time.Duration) (time.Duration, bool) { + start := time.Now() + if duration <= 0 { + return time.Since(start), false + } + + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-timer.C: + return time.Since(start), false + case <-ctx.Done(): + return time.Since(start), true + } } // handleRateLimitWithUnitReporter is a wrapper around handleRateLimit that includes unit reporting @@ -930,23 +1596,38 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporter return nil } -func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, reporter sources.UnitReporter) error { - opts := &github.ListOptions{ - PerPage: membersAppPagination, - } - +func (s *Source) addMembersByApp(ctx context.Context, connector *appConnector, reporter sources.UnitReporter) error { // TODO: Check rate limit for this call. - installs, _, err := installationClient.Apps.ListInstallations(ctx, opts) + installs, err := s.getAppInstallations(ctx, connector) if err != nil { return fmt.Errorf("could not enumerate installed orgs: %w", err) } - for _, org := range installs { - if org.Account.GetType() != "Organization" { + for _, install := range installs { + if install.GetAccount().GetType() != "Organization" { continue } - if err := s.addMembersByOrg(ctx, *org.Account.Login, reporter); err != nil { - return err + org := install.GetAccount().GetLogin() + installCtx := context.WithValues(ctx, "org", org, "installation_id", install.GetID()) + + // Create a per-installation API client so that the token is scoped to + // this org. GitHub App installation tokens only bypass IP allowlists for + // their own org; using a cross-org token causes 403s. + client, err := connector.APIClientForInstallation(install.GetID()) + if err != nil { + installCtx.Logger().Error(err, "could not create API client for installation") + if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", install.GetID(), err)); err != nil { + return err + } + continue + } + + if err := s.addMembersByOrgWithClient(installCtx, client, org, reporter); err != nil { + installCtx.Logger().Error(err, "Unable to add members for org") + if err := reporter.UnitErr(installCtx, fmt.Errorf("could not add members for org %q (installation %d): %w", org, install.GetID(), err)); err != nil { + return err + } + continue } } @@ -1028,6 +1709,10 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string, reporter source } func (s *Source) addMembersByOrg(ctx context.Context, org string, reporter sources.UnitReporter) error { + return s.addMembersByOrgWithClient(ctx, s.connector.APIClient(), org, reporter) +} + +func (s *Source) addMembersByOrgWithClient(ctx context.Context, client *github.Client, org string, reporter sources.UnitReporter) error { opts := &github.ListMembersOptions{ PublicOnly: false, ListOptions: github.ListOptions{ @@ -1037,7 +1722,7 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string, reporter sourc logger := ctx.Logger().WithValues("org", org) for { - members, res, err := s.connector.APIClient().Organizations.ListMembers(ctx, org, opts) + members, res, err := client.Organizations.ListMembers(ctx, org, opts) if s.handleRateLimitWithUnitReporter(ctx, reporter, err) { continue } @@ -1095,15 +1780,24 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep cutoffTime = &t } + apiClient, err := s.connector.APIClientForRepo(repoPath) + if err != nil { + return err + } + if s.includeGistComments && isGistUrl(urlParts) && !s.ignoreGists { - return s.processGistComments(ctx, urlString, urlParts, repoInfo, reporter, cutoffTime) + return s.processGistComments(ctx, apiClient, urlString, urlParts, repoInfo, reporter, cutoffTime) } else if s.includeIssueComments || s.includePRComments { // if we need to use graphql api for repo issues, prs and comments if feature.UseGithubGraphQLAPI.Load() { - return s.processRepoIssueandPRsWithCommentsGraphql(ctx, repoInfo, reporter, cutoffTime) + graphqlClient, err := s.connector.GraphQLClientForRepo(ctx, repoPath) + if err != nil { + return err + } + return s.processRepoIssueandPRsWithCommentsGraphql(ctx, graphqlClient, repoInfo, reporter, cutoffTime) } - return s.processIssueandPRsWithCommentsREST(ctx, repoInfo, reporter, cutoffTime) + return s.processIssueandPRsWithCommentsREST(ctx, apiClient, repoInfo, reporter, cutoffTime) } return nil @@ -1170,7 +1864,7 @@ func getRepoURLParts(repoURLString string) (string, []string, error) { const initialPage = 1 // page to start listing from -func (s *Source) processGistComments(ctx context.Context, gistURL string, urlParts []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) processGistComments(ctx context.Context, apiClient *github.Client, gistURL string, urlParts []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { ctx.Logger().V(2).Info("Scanning GitHub Gist comments") // GitHub Gist URL. @@ -1181,7 +1875,7 @@ func (s *Source) processGistComments(ctx context.Context, gistURL string, urlPar Page: initialPage, } for { - comments, _, err := s.connector.APIClient().Gists.ListComments(ctx, gistID, options) + comments, _, err := apiClient.Gists.ListComments(ctx, gistID, options) if s.handleRateLimitWithChunkReporter(ctx, reporter, err) { continue } @@ -1279,25 +1973,25 @@ var ( ) func (s *Source) processIssueandPRsWithCommentsREST( - ctx context.Context, repoInfo repoInfo, + ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time, ) error { if s.includeIssueComments { ctx.Logger().V(2).Info("Scanning issues") - if err := s.processIssues(ctx, repoInfo, reporter); err != nil { + if err := s.processIssues(ctx, apiClient, repoInfo, reporter); err != nil { return err } - if err := s.processIssueComments(ctx, repoInfo, reporter, cutoffTime); err != nil { + if err := s.processIssueComments(ctx, apiClient, repoInfo, reporter, cutoffTime); err != nil { return err } } if s.includePRComments { ctx.Logger().V(2).Info("Scanning pull requests") - if err := s.processPRs(ctx, repoInfo, reporter); err != nil { + if err := s.processPRs(ctx, apiClient, repoInfo, reporter); err != nil { return err } - if err := s.processPRComments(ctx, repoInfo, reporter, cutoffTime); err != nil { + if err := s.processPRComments(ctx, apiClient, repoInfo, reporter, cutoffTime); err != nil { return err } } @@ -1306,23 +2000,23 @@ func (s *Source) processIssueandPRsWithCommentsREST( } func (s *Source) processRepoIssueandPRsWithCommentsGraphql( - ctx context.Context, repoInfo repoInfo, + ctx context.Context, graphqlClient *githubv4.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time, ) error { if s.includeIssueComments { ctx.Logger().V(2).Info("Scanning issues") - if err := s.processIssuesWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil { + if err := s.processIssuesWithComments(ctx, graphqlClient, repoInfo, reporter, cutoffTime); err != nil { return err } } if s.includePRComments { ctx.Logger().V(2).Info("Scanning pull requests") - if err := s.processPRWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil { + if err := s.processPRWithComments(ctx, graphqlClient, repoInfo, reporter, cutoffTime); err != nil { return err } - if err := s.processReviewThreads(ctx, repoInfo, reporter, cutoffTime); err != nil { + if err := s.processReviewThreads(ctx, graphqlClient, repoInfo, reporter, cutoffTime); err != nil { return err } } @@ -1330,7 +2024,7 @@ func (s *Source) processRepoIssueandPRsWithCommentsGraphql( return nil } -func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter) error { +func (s *Source) processIssues(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter) error { bodyTextsOpts := &github.IssueListByRepoOptions{ Sort: sortType, Direction: directionType, @@ -1342,7 +2036,7 @@ func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter } for { - issues, _, err := s.connector.APIClient().Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts) + issues, _, err := apiClient.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts) if s.handleRateLimitWithChunkReporter(ctx, reporter, err) { continue } @@ -1355,7 +2049,7 @@ func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter return err } - bodyTextsOpts.ListOptions.Page++ + bodyTextsOpts.Page++ if len(issues) < defaultPagination { break @@ -1400,7 +2094,7 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g return nil } -func (s *Source) processIssueComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) processIssueComments(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { issueOpts := &github.IssueListCommentsOptions{ Sort: &sortType, Direction: &directionType, @@ -1411,7 +2105,7 @@ func (s *Source) processIssueComments(ctx context.Context, repoInfo repoInfo, re } for { - issueComments, _, err := s.connector.APIClient().Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts) + issueComments, _, err := apiClient.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts) if s.handleRateLimitWithChunkReporter(ctx, reporter, err) { continue } @@ -1423,7 +2117,7 @@ func (s *Source) processIssueComments(ctx context.Context, repoInfo repoInfo, re return err } - issueOpts.ListOptions.Page++ + issueOpts.Page++ if len(issueComments) < defaultPagination { break } @@ -1467,7 +2161,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm return nil } -func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter) error { +func (s *Source) processPRs(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter) error { prOpts := &github.PullRequestListOptions{ Sort: sortType, Direction: directionType, @@ -1479,7 +2173,7 @@ func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, reporter sou } for { - prs, _, err := s.connector.APIClient().PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts) + prs, _, err := apiClient.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts) if s.handleRateLimitWithChunkReporter(ctx, reporter, err) { continue } @@ -1491,7 +2185,7 @@ func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, reporter sou return err } - prOpts.ListOptions.Page++ + prOpts.Page++ if len(prs) < defaultPagination { break @@ -1500,7 +2194,7 @@ func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, reporter sou return nil } -func (s *Source) processPRComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) processPRComments(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { prOpts := &github.PullRequestListCommentsOptions{ Sort: sortType, Direction: directionType, @@ -1511,7 +2205,7 @@ func (s *Source) processPRComments(ctx context.Context, repoInfo repoInfo, repor } for { - prComments, _, err := s.connector.APIClient().PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts) + prComments, _, err := apiClient.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts) if s.handleRateLimitWithChunkReporter(ctx, reporter, err) { continue } @@ -1523,7 +2217,7 @@ func (s *Source) processPRComments(ctx context.Context, repoInfo repoInfo, repor return err } - prOpts.ListOptions.Page++ + prOpts.Page++ if len(prComments) < defaultPagination { break @@ -1642,15 +2336,43 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget, return fmt.Errorf("invalid GitHub URL") } + repoURL, err := s.repoURLFromTargetMetadata(meta) + if err != nil { + return err + } + targetIsWiki := isWikiLink(meta.GetLink()) + if targetIsWiki { + repoURL, err = repoCloneURLForTargetEndpoint(s.conn.GetEndpoint(), repoURL) + if err != nil { + return err + } + } + if connector, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations { + if err := s.mapReposToInstallations(ctx, connector, []string{repoURL}); err != nil { + return err + } + } + if targetIsWiki { + wikiURL, ok := wikiCloneURLForRepo(repoURL) + if !ok { + return fmt.Errorf("could not derive wiki clone URL from target repository %q", repoURL) + } + return s.scanWikiTarget(ctx, wikiURL, meta, &chunkSkel, reporter) + } + apiClient, err := s.connector.APIClientForRepo(repoURL) + if err != nil { + return err + } + if meta.GetFile() == "" && meta.GetCommit() != "" { ctx := context.WithValues(ctx, "commit_hash", meta.GetCommit()) ctx.Logger().V(2).Info("secret metadata has no file; scanning commit metadata instead") - return s.scanCommitMetadata(ctx, segments[1], segments[2], meta, &chunkSkel, reporter) + return s.scanCommitMetadata(ctx, apiClient, segments[1], segments[2], meta, &chunkSkel, reporter) } // else try downloading the file content to scan - readCloser, resp, err := s.connector.APIClient().Repositories.DownloadContents( + readCloser, resp, err := apiClient.Repositories.DownloadContents( ctx, segments[1], segments[2], @@ -1659,10 +2381,33 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget, // As of this writing, if the returned readCloser is not nil, it's just the Body of the returned github.Response, so // there's no need to independently close it. if resp != nil && resp.Body != nil { - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } if err != nil { - return fmt.Errorf("could not download file for scan: %w", err) + // DownloadContents locates the file by listing its parent directory, + // and the contents API caps listings at 1000 entries, so it can fail + // for files that exist. Retry with an exact-path lookup, which also + // serves as the existence probe when it fails. + retryCloser, retryResp, retryErr := s.downloadContentsByPath( + ctx, apiClient, segments[1], segments[2], meta.GetFile(), meta.GetCommit()) + if retryResp != nil && retryResp.Response != nil && retryResp.Body != nil { + defer func() { _ = retryResp.Body.Close() }() + } + if retryErr != nil { + wrapped := fmt.Errorf("could not download file for scan: %w", retryErr) + // The exact-path lookup 404ing is authoritative for the path, but + // GitHub also returns 404 for existing resources the credentials + // cannot see, so only classify the target as gone when the + // repository itself is still reachable with the same client. + if retryResp != nil && retryResp.Response != nil && + retryResp.StatusCode == http.StatusNotFound && + s.repoReachable(ctx, apiClient, segments[1], segments[2]) { + return &sources.TargetNotFoundError{Err: wrapped} + } + return wrapped + } + fileCtx := context.WithValues(ctx, "path", meta.GetFile()) + return handlers.HandleFile(fileCtx, retryCloser, &chunkSkel, reporter) } if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected HTTP response status when trying to download file for scan: %v", resp.Status) @@ -1672,11 +2417,93 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget, return handlers.HandleFile(fileCtx, readCloser, &chunkSkel, reporter) } -func (s *Source) scanCommitMetadata(ctx context.Context, owner, repo string, meta *source_metadatapb.Github, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) error { +// downloadContentsByPath fetches a file the same way DownloadContents does, +// but resolves it with an exact-path contents lookup instead of searching a +// directory listing, so it is immune to the 1000-entry listing cap. On +// failure the returned response is always the exact-path lookup's, whose 404 +// is an authoritative statement about the path at that ref; a failure of the +// download itself must not be mistaken for the target being gone, because +// the lookup just proved it exists. +func (s *Source) downloadContentsByPath(ctx context.Context, apiClient *github.Client, owner, repo, filePath, ref string) (io.ReadCloser, *github.Response, error) { + fileContent, _, resp, err := apiClient.Repositories.GetContents( + ctx, owner, repo, filePath, &github.RepositoryContentGetOptions{Ref: ref}) + if err != nil { + return nil, resp, err + } + if fileContent == nil || fileContent.GetDownloadURL() == "" { + return nil, resp, fmt.Errorf("no download link found for %s", filePath) + } + dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, fileContent.GetDownloadURL(), nil) + if err != nil { + return nil, resp, err + } + dlResp, err := apiClient.Client().Do(dlReq) + if err != nil { + return nil, resp, err + } + if dlResp.StatusCode != http.StatusOK { + _ = dlResp.Body.Close() + return nil, resp, fmt.Errorf("unexpected HTTP response status when trying to download file for scan: %v", dlResp.Status) + } + return dlResp.Body, &github.Response{Response: dlResp}, nil +} + +// repoReachable reports whether the repository is still visible with the +// same client. +func (s *Source) repoReachable(ctx context.Context, apiClient *github.Client, owner, repo string) bool { + _, resp, err := apiClient.Repositories.Get(ctx, owner, repo) + return err == nil && resp != nil && resp.StatusCode == http.StatusOK +} + +func (s *Source) scanWikiTarget(ctx context.Context, wikiURL string, meta *source_metadatapb.Github, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) error { + if meta.GetCommit() == "" { + return fmt.Errorf("wiki target has no commit") + } + + path, repo, err := s.cloneRepo(ctx, wikiURL) + if err != nil { + return fmt.Errorf("could not clone wiki for targeted scan: %w", err) + } + if !s.conn.GetPrintLegacyJson() { + if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.conn.NoCleanup && s.conn.GetClonePath() != "") { + defer func() { _ = os.RemoveAll(path) }() + } + } + + commit, err := repo.CommitObject(plumbing.NewHash(meta.GetCommit())) + if err != nil { + return fmt.Errorf("could not fetch wiki commit for targeted scan: %w", err) + } + + if meta.GetFile() == "" { + var sb strings.Builder + sb.WriteString(commit.Author.Email) + sb.WriteString("\n") + sb.WriteString(commit.Committer.Email) + sb.WriteString("\n") + sb.WriteString(commit.Message) + return handlers.HandleFile(ctx, io.NopCloser(strings.NewReader(sb.String())), chunkSkel, reporter) + } + + file, err := commit.File(meta.GetFile()) + if err != nil { + return fmt.Errorf("could not find wiki file for targeted scan: %w", err) + } + readCloser, err := file.Reader() + if err != nil { + return fmt.Errorf("could not read wiki file for targeted scan: %w", err) + } + defer func() { _ = readCloser.Close() }() + + fileCtx := context.WithValues(ctx, "path", meta.GetFile()) + return handlers.HandleFile(fileCtx, readCloser, chunkSkel, reporter) +} + +func (s *Source) scanCommitMetadata(ctx context.Context, apiClient *github.Client, owner, repo string, meta *source_metadatapb.Github, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) error { // fetch the commit - commit, resp, err := s.connector.APIClient().Repositories.GetCommit(ctx, owner, repo, meta.GetCommit(), nil) + commit, resp, err := apiClient.Repositories.GetCommit(ctx, owner, repo, meta.GetCommit(), nil) if resp != nil && resp.Body != nil { - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() } if err != nil { return fmt.Errorf("could not fetch commit for metadata scan: %w", err) @@ -1701,8 +2528,21 @@ func (s *Source) scanCommitMetadata(ctx context.Context, owner, repo string, met } func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error { - repoURL, _ := unit.SourceUnitID() + repoURL, kind := unit.SourceUnitID() ctx = context.WithValue(ctx, "repo", repoURL) + + // If the unit has an installation ID, use it. Fetching it is slow, + // specifically it stalls large multi-org scans on rate limits (INT-790). + + // [CG] This is pretty ugly; if you can clean it up please do (I failed). + if ac, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations && kind == "repo" { + if ru, ok := unit.(RepoUnit); ok && ru.InstallationID > 0 { + ac.setRepoInstallationForRepoName(repoURL, ru.Name, ru.InstallationID) + } else if err := s.mapReposToInstallations(ctx, ac, []string{repoURL}); err != nil { + return err + } + } + // ChunkUnit is not guaranteed to be called from Enumerate, so we must // check and fetch the repoInfoCache for this repo. repoURL, err := s.ensureRepoInfoCache(ctx, repoURL, &chunkErrorReporter{reporter: reporter}) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 1bbed6305358..5729f48d2d25 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -65,13 +65,93 @@ func TestSource_Token(t *testing.T) { s.Init(ctx, "github integration test source", 0, 0, false, conn, 1) s.filteredRepoCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](), nil, nil) - err = s.enumerateWithApp(ctx, s.connector.(*appConnector).InstallationClient(), noopReporter()) + err = s.enumerateWithApp(ctx, s.connector.(*appConnector), noopReporter()) assert.NoError(t, err) _, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git") assert.NoError(t, err) } +func TestSource_ExcludeArchived(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + + secret, err := common.GetTestSecret(ctx) + if err != nil { + t.Fatal(fmt.Errorf("failed to access secret: %v", err)) + } + + githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW") + githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New) + if err != nil { + t.Fatal(err) + } + githubPrivateKeyNew := string(githubPrivateKeyBytesNew) + githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") + githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") + + srcWithExclude := &sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + ExcludeArchived: true, + } + connWithExclude, err := anypb.New(srcWithExclude) + if err != nil { + panic(err) + } + + sWithExclude := Source{ + conn: srcWithExclude, + memberCache: map[string]struct{}{}, + repoInfoCache: newRepoInfoCache(), + } + sWithExclude.Init(ctx, "github integration test source with exclude", 0, 0, false, connWithExclude, 1) + sWithExclude.filteredRepoCache = sWithExclude.newFilteredRepoCache(ctx, simple.NewCache[string](), nil, nil) + + err = sWithExclude.enumerateWithApp(ctx, sWithExclude.connector.(*appConnector), noopReporter()) + assert.NoError(t, err) + + srcWithoutExclude := &sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + ExcludeArchived: false, + } + connWithoutExclude, err := anypb.New(srcWithoutExclude) + if err != nil { + panic(err) + } + + sWithoutExclude := Source{ + conn: srcWithoutExclude, + memberCache: map[string]struct{}{}, + repoInfoCache: newRepoInfoCache(), + } + sWithoutExclude.Init(ctx, "github integration test source without exclude", 0, 0, false, connWithoutExclude, 1) + sWithoutExclude.filteredRepoCache = sWithoutExclude.newFilteredRepoCache(ctx, simple.NewCache[string](), nil, nil) + + err = sWithoutExclude.enumerateWithApp(ctx, sWithoutExclude.connector.(*appConnector), noopReporter()) + assert.NoError(t, err) + + assert.Less(t, sWithExclude.filteredRepoCache.Count(), sWithoutExclude.filteredRepoCache.Count(), + "ExcludeArchived should result in strictly fewer repos (test org must have at least one archived repo). WithExclude: %d, WithoutExclude: %d", + sWithExclude.filteredRepoCache.Count(), sWithoutExclude.filteredRepoCache.Count()) + + t.Logf("Repos with ExcludeArchived=true: %d", sWithExclude.filteredRepoCache.Count()) + t.Logf("Repos with ExcludeArchived=false: %d", sWithoutExclude.filteredRepoCache.Count()) +} + func TestSource_ScanComments(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -676,7 +756,7 @@ func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) so if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" { return nil } - return sources.MatchError + return sources.ErrMatch } return nil } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 50e8f0c9efb1..29911c07822e 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/http" + "net/http/httptest" "net/url" "reflect" "slices" @@ -28,6 +29,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) @@ -302,7 +304,7 @@ func TestAddMembersByApp(t *testing.T) { Get("/app/installations"). Reply(200). JSON([]map[string]any{ - {"account": map[string]string{"login": "super-secret-org", "type": "Organization"}}, + {"id": 1337, "account": map[string]string{"login": "super-secret-org", "type": "Organization"}}, }) gock.New("https://api.github.com"). Post("/app/installations/1337/access_tokens"). @@ -326,7 +328,7 @@ func TestAddMembersByApp(t *testing.T) { AppId: "4141", }, }}) - err := s.addMembersByApp(context.Background(), s.connector.(*appConnector).InstallationClient(), noopReporter()) + err := s.addMembersByApp(context.Background(), s.connector.(*appConnector), noopReporter()) assert.Nil(t, err) assert.Equal(t, 3, len(s.memberCache)) _, ok := s.memberCache["ssm1"] @@ -364,6 +366,82 @@ func TestAddReposByApp(t *testing.T) { assert.True(t, gock.IsDone()) } +// TestAppConnector_EnterpriseBaseURL verifies that GitHub App authentication +// correctly sets the BaseURL on the internal AppsTransport for token refresh. +// This is critical for GitHub Enterprise Server and GHEC with Data Residency. +// Without this fix, token refresh requests would go to api.github.com instead +// of the configured enterprise endpoint. +func TestAppConnector_EnterpriseBaseURL(t *testing.T) { + privateKey := createPrivateKey() + enterpriseEndpoint := "https://api.example.ghe.com" + + connector, err := NewAppConnector(enterpriseEndpoint, &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, false) + require.NoError(t, err) + + appConn, ok := connector.(*appConnector) + require.True(t, ok, "connector should be an appConnector") + + // Get the HTTP client's transport, which should be a ghinstallation.Transport + transport := appConn.APIClient().Client().Transport + + // Use reflection to access the Transport's BaseURL field + transportVal := reflect.ValueOf(transport).Elem() + baseURLField := transportVal.FieldByName("BaseURL") + require.True(t, baseURLField.IsValid(), "Transport should have a BaseURL field") + assert.Equal(t, enterpriseEndpoint, baseURLField.String(), + "Transport.BaseURL should be set to enterprise endpoint") + + // Use reflection to access the internal appsTransport and verify its BaseURL + // This is the critical check - the internal AppsTransport is used for token refresh + appsTransportField := transportVal.FieldByName("appsTransport") + require.True(t, appsTransportField.IsValid(), "Transport should have an appsTransport field") + + // Access the BaseURL of the internal AppsTransport + appsTransportVal := appsTransportField.Elem() + appsBaseURLField := appsTransportVal.FieldByName("BaseURL") + require.True(t, appsBaseURLField.IsValid(), "AppsTransport should have a BaseURL field") + assert.Equal(t, enterpriseEndpoint, appsBaseURLField.String(), + "AppsTransport.BaseURL should be set to enterprise endpoint") +} + +// TestValidate_ScanAllInstallationsWithoutDefaultInstallation covers the +// scanAllInstallations-without-githubApp.installationId case: there is no +// default installation token to call RateLimit with (see +// appConnector.HasDefaultInstallation), so Validate must fall back to +// checking App credentials via the app-level (JWT) client instead of +// erroring against a non-existent installation ID 0. +func TestValidate_ScanAllInstallationsWithoutDefaultInstallation(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/app/installations"). + Reply(200). + JSON([]map[string]any{}) + gock.New("https://api.github.com"). + Get("^/app$"). + Reply(200). + JSON(map[string]any{"id": 4141}) + + s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: createPrivateKey(), + AppId: "4141", + }, + }}) + + errs := s.Validate(context.Background()) + assert.Empty(t, errs) + assert.False(t, gock.HasUnmatchedRequest()) + assert.True(t, gock.IsDone()) +} + func TestAddOrgsByUser(t *testing.T) { defer gock.Off() @@ -478,12 +556,17 @@ func TestNormalizeRepo(t *testing.T) { assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") - // Test case 5: Org/repo format (should convert to full URL) + // Test case 5: SCP-style SSH URL + result, err = source.normalizeRepo("git@github.com:org/repo.git") + assert.NoError(t, err) + assert.Equal(t, "https://github.com/org/repo.git", result) + + // Test case 6: Org/repo format (should convert to full URL) result, err = source.normalizeRepo("org/repo") assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") - // Test case 6: Invalid format (no protocol, no slash) + // Test case 7: Invalid format (no protocol, no slash) _, err = source.normalizeRepo("invalid") assert.Error(t, err) assert.Contains(t, err.Error(), "no repositories found") @@ -505,6 +588,11 @@ func TestNormalizeRepo_Enterprise(t *testing.T) { endpoint: "https://example.com/api/v3", wantResult: "https://example.com/org/repo.git", }, + { + name: "host with path without scheme", + endpoint: "example.com/api/v3", + wantResult: "https://example.com/org/repo.git", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -522,6 +610,7 @@ func TestNormalizeRepo_Enterprise(t *testing.T) { } func TestHandleRateLimit(t *testing.T) { + resetRateLimitForTest(t) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) ctx := context.Background() assert.False(t, s.handleRateLimit(ctx, nil)) @@ -558,6 +647,36 @@ func TestHandleRateLimit(t *testing.T) { assert.True(t, s.handleRateLimit(ctx, err)) } +func TestHandleRateLimitStopsSleepingWhenContextCancelled(t *testing.T) { + resetRateLimitForTest(t) + s := &Source{name: "test - github"} + retryAfter := time.Hour + err := &github.AbuseRateLimitError{ + RetryAfter: &retryAfter, + Message: "secondary rate limit", + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + assert.False(t, s.handleRateLimit(ctx, err)) + assert.Less(t, time.Since(start), 200*time.Millisecond) +} + +func resetRateLimitForTest(t *testing.T) { + t.Helper() + rateLimitMu.Lock() + previous := rateLimitResumeTime + rateLimitResumeTime = time.Time{} + rateLimitMu.Unlock() + + t.Cleanup(func() { + rateLimitMu.Lock() + rateLimitResumeTime = previous + rateLimitMu.Unlock() + }) +} + func TestEnumerateUnauthenticated(t *testing.T) { defer gock.Off() @@ -884,186 +1003,1181 @@ func TestEnumerateWithApp(t *testing.T) { }, }, }) - err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector).InstallationClient(), noopReporter()) + err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector), noopReporter()) assert.Nil(t, err) assert.Equal(t, 0, len(s.repos)) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } -// This only tests the resume info slice portion of setProgressCompleteWithRepo. -func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) { - tests := []struct { - startingResumeInfoSlice []string - repoURL string - wantResumeInfoSlice []string - }{ - { - startingResumeInfoSlice: []string{}, - repoURL: "a", - wantResumeInfoSlice: []string{"a"}, - }, - { - startingResumeInfoSlice: []string{"b"}, - repoURL: "a", - wantResumeInfoSlice: []string{"a", "b"}, +func TestEnumerateWithAppScanAllInstallationsIncludesConfiguredInstallation(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[ + {"id":1337,"account":{"login":"default-org","type":"Organization"}}, + {"id":2448,"account":{"login":"other-org","type":"Organization"}} + ]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + auth := r.Header.Get("Authorization") + switch { + case strings.Contains(auth, "token-1337"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"default-org/repo","clone_url":"https://github.com/default-org/repo.git","owner":{"login":"default-org","type":"Organization"},"size":1} + ]}`)) + case strings.Contains(auth, "token-2448"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + default: + http.Error(w, "unexpected installation token", http.StatusUnauthorized) + } + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, }, - } + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) - s := &Source{ - repos: []string{}, - } + connector := s.connector.(*appConnector) + err := s.enumerateWithApp(context.Background(), connector, noopReporter()) + require.NoError(t, err) - for _, tt := range tests { - s.resumeInfoSlice = tt.startingResumeInfoSlice - s.setProgressCompleteWithRepo(0, 0, tt.repoURL) - if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) { - t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice) - } - } + assert.Equal(t, int64(1337), connector.repoInstallationMap["https://github.com/default-org/repo.git"]) + assert.Equal(t, int64(2448), connector.repoInstallationMap["https://github.com/other-org/repo.git"]) + assert.True(t, s.filteredRepoCache.Exists("default-org/repo")) + assert.True(t, s.filteredRepoCache.Exists("other-org/repo")) } -func Test_setProgressCompleteWithRepo_Progress(t *testing.T) { - repos := []string{"a", "b", "c", "d", "e"} - tests := map[string]struct { - repos []string - index int - offset int - wantPercentComplete int64 - wantSectionsCompleted int32 - wantSectionsRemaining int32 - }{ - "starting from the beginning, no offset": { - repos: repos, - index: 0, - offset: 0, - wantPercentComplete: 0, - wantSectionsCompleted: 0, - wantSectionsRemaining: 5, - }, - "resume from the third, offset 2": { - repos: repos[2:], - index: 0, - offset: 2, - wantPercentComplete: 40, - wantSectionsCompleted: 2, - wantSectionsRemaining: 5, - }, - "resume from the third, on last repo, offset 2": { - repos: repos[2:], - index: 2, - offset: 2, - wantPercentComplete: 80, - wantSectionsCompleted: 4, - wantSectionsRemaining: 5, - }, - } +func TestEnumerateWithAppScanAllInstallationsMapsExplicitReposBeforeMetadataFetch(t *testing.T) { + privateKey := createPrivateKey() - for _, tt := range tests { - s := &Source{ - repos: tt.repos, + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[ + {"id":1337,"account":{"login":"default-org","type":"Organization"}}, + {"id":2448,"account":{"login":"other-org","type":"Organization"}} + ]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + auth := r.Header.Get("Authorization") + switch { + case strings.Contains(auth, "token-1337"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"default-org/repo","clone_url":"https://github.com/default-org/repo.git","owner":{"login":"default-org","type":"Organization"},"size":1} + ]}`)) + case strings.Contains(auth, "token-2448"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + default: + http.Error(w, "unexpected installation token", http.StatusUnauthorized) + } + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + if !strings.Contains(r.Header.Get("Authorization"), "token-2448") { + http.Error(w, "wrong installation token", http.StatusForbidden) + return + } + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) } + })) + defer server.Close() - s.setProgressCompleteWithRepo(tt.index, tt.offset, "") - gotProgress := s.GetProgress() - if gotProgress.PercentComplete != tt.wantPercentComplete { - t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete) - } - if gotProgress.SectionsCompleted != tt.wantSectionsCompleted { - t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted) - } - if gotProgress.SectionsRemaining != tt.wantSectionsRemaining { - t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining) - } - } + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{"https://github.com/other-org/repo.git"}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + err := s.Enumerate(context.Background(), noopReporter()) + require.NoError(t, err) + + connector := s.connector.(*appConnector) + assert.Equal(t, int64(2448), connector.repoInstallationMap["https://github.com/other-org/repo.git"]) + info, ok := s.repoInfoCache.get("https://github.com/other-org/repo.git") + require.True(t, ok) + assert.Equal(t, "other-org/repo", info.fullName) } -func Test_scan_SetProgressComplete(t *testing.T) { - testCases := []struct { - name string - repos []string - wantComplete bool - wantErr bool - }{ - { - name: "no repos", - wantComplete: true, - }, - { - name: "one valid repo", - repos: []string{"https://github.com/super-secret-user/super-secret-repo.git"}, - wantComplete: true, - wantErr: true, +func TestMapExplicitReposToInstallationsErrorsForUnmatchedRepos(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"different","full_name":"other-org/different","clone_url":"https://github.com/other-org/different.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{"https://github.com/other-org/missing.git"}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, }, - } + }) + err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) + require.Error(t, err) + assert.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation") + assert.Contains(t, err.Error(), "https://github.com/other-org/missing.git") +} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - src := initTestSource(&sourcespb.GitHub{ - Repositories: tc.repos, - Credential: &sourcespb.GitHub_Unauthenticated{}, - }) - src.jobPool = &errgroup.Group{} +func TestScanAllInstallationsInitErrorsWhenInstallationsCannotBeListed(t *testing.T) { + privateKey := createPrivateKey() - _ = src.scan(context.Background(), nil) - if !tc.wantErr { - assert.Equal(t, "", src.GetProgress().EncodedResumeInfo) - } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") - gotComplete := src.GetProgress().PercentComplete == 100 - if gotComplete != tc.wantComplete { - t.Errorf("got: %v, want: %v", gotComplete, tc.wantComplete) - } - }) - } + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + http.Error(w, "installation list failed", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + + err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) + require.Error(t, err) + assert.Contains(t, err.Error(), "could not list installations") } -func TestGetRepoURLParts(t *testing.T) { - repoURLs := []string{ - "https://github.com/trufflesecurity/trufflehog.git", - "git+https://github.com/trufflesecurity/trufflehog.git", - "ssh://github.com/trufflesecurity/trufflehog.git", - "ssh://git@github.com/trufflesecurity/trufflehog.git", - "git+ssh://git@github.com/trufflesecurity/trufflehog.git", - "git://github.com/trufflesecurity/trufflehog.git", - } - expected := []string{"github.com", "trufflesecurity", "trufflehog"} - for _, tt := range repoURLs { - _, parts, err := getRepoURLParts(tt) - if err != nil { - t.Fatalf("failed: %v", err) +func TestMapExplicitReposToInstallationsStopsAfterReposFound(t *testing.T) { + privateKey := createPrivateKey() + var listInstallationsCalls, unrelatedRepoListCalls int + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + listInstallationsCalls++ + _, _ = w.Write([]byte(`[ + {"id":1337,"account":{"login":"default-org","type":"Organization"}}, + {"id":2448,"account":{"login":"other-org","type":"Organization"}}, + {"id":3559,"account":{"login":"unrelated-org","type":"Organization"}} + ]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + auth := r.Header.Get("Authorization") + switch { + case strings.Contains(auth, "token-1337"): + _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) + case strings.Contains(auth, "token-2448"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + case strings.Contains(auth, "token-3559"): + unrelatedRepoListCalls++ + http.Error(w, "unrelated installation failed", http.StatusInternalServerError) + default: + http.Error(w, "unexpected installation token", http.StatusUnauthorized) + } + default: + http.NotFound(w, r) } - assert.Equal(t, expected, parts) - } + })) + defer server.Close() - gistURLs := map[string][]string{ - // Gists - "ssh://github.com/6df198861306313246466d23aa4102aa.git": nil, - "ssh://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"}, - "https://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"}, - "https://gist.github.com/john-smith/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "john-smith", "6df198861306313246466d23aa4102aa"}, - "ssh://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"}, - "https://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"}, - "https://github.contoso.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "john-smith", "6df198861306313246466d23aa4102aa"}, - "https://github.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": nil, - } - for tt, expected := range gistURLs { - _, parts, err := getRepoURLParts(tt) - if err != nil { - if expected == nil { - continue + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{"https://github.com/other-org/repo.git"}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + require.NoError(t, s.mapExplicitReposToInstallations(context.Background(), connector)) + require.NoError(t, s.mapExplicitReposToInstallations(context.Background(), connector)) + + installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) + assert.Equal(t, 1, listInstallationsCalls) + assert.Equal(t, 0, unrelatedRepoListCalls) +} + +func TestScanAllInstallationsReusesInstallationsLoadedDuringInit(t *testing.T) { + privateKey := createPrivateKey() + var listInstallationsCalls int + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + listInstallationsCalls++ + _, _ = w.Write([]byte(`[ + {"id":1337,"account":{"login":"default-org","type":"Organization"}}, + {"id":2448,"account":{"login":"other-org","type":"Organization"}} + ]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + if strings.Contains(r.Header.Get("Authorization"), "token-2448") { + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + return } - t.Fatalf("failed: %v", err) + _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/orgs/") && strings.HasSuffix(r.URL.Path, "/members"): + _, _ = w.Write([]byte(`[]`)) + default: + http.NotFound(w, r) } - assert.Equal(t, expected, parts) - } -} + })) + defer server.Close() -func TestGetRepoURLPartsWithTrailingHyphen(t *testing.T) { - // Test for https://github.com/trufflesecurity/trufflehog/issues/4679 - // Repository names ending with a hyphen should be preserved correctly. - testCases := []struct { - name string + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{"https://github.com/other-org/repo.git"}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + require.NoError(t, s.enumerateAllInstallationRepos(context.Background(), connector, noopReporter())) + require.NoError(t, s.addMembersByApp(context.Background(), connector, noopReporter())) + assert.Equal(t, 1, listInstallationsCalls) +} + +func TestMapExplicitReposToInstallationsMatchesSSHRepoURL(t *testing.T) { + privateKey := createPrivateKey() + const sshRepoURL = "ssh://git@github.com/other-org/repo.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{sshRepoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + installationID, mapped := connector.installationIDForRepo(sshRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) +} + +func TestMapExplicitReposToInstallationsMatchesSCPStyleSSHRepoURL(t *testing.T) { + privateKey := createPrivateKey() + const sshRepoURL = "git@github.com:other-org/repo.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{sshRepoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + installationID, mapped := connector.installationIDForRepo(sshRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) +} + +func TestMapExplicitReposToInstallationsRejectsHostMismatch(t *testing.T) { + privateKey := createPrivateKey() + const attackerRepoURL = "https://attacker.example/other-org/repo.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{attackerRepoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) + require.Error(t, err) + assert.Contains(t, err.Error(), "configured repos were not found") +} + +// A repo absent from every installation listing but readable with the +// default installation token (e.g. a public repo, or one the listing missed) +// maps to the default installation instead of failing the scan. Repos the +// default token cannot read are still rejected (see the preceding test and +// TestScanAllInstallationsInaccessibleRepoStillFailsMapping). +func TestMapExplicitReposToInstallationsFallsBackToDefaultForReadableRepo(t *testing.T) { + privateKey := createPrivateKey() + const repoURL = "https://github.com/other-org/repo.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{repoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) + require.NoError(t, err) + + connector := s.connector.(*appConnector) + installationID, mapped := connector.installationIDForRepo(repoURL) + require.True(t, mapped) + assert.Equal(t, int64(1337), installationID) +} + +func TestMapExplicitReposToInstallationsMapsWikiRepoURL(t *testing.T) { + privateKey := createPrivateKey() + const wikiRepoURL = "https://github.com/other-org/repo.wiki.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{wikiRepoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) + installationID, mapped = connector.installationIDForRepo(wikiRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) +} + +func TestMapExplicitReposToInstallationsPrefersRealWikiSuffixedRepo(t *testing.T) { + privateKey := createPrivateKey() + const parentRepoURL = "https://github.com/other-org/repo.git" + const wikiSuffixedRepoURL = "https://github.com/other-org/repo.wiki.git" + const wikiOfWikiSuffixedRepoURL = "https://github.com/other-org/repo.wiki.wiki.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":2,"repositories":[ + {"name":"repo.wiki","full_name":"other-org/repo.wiki","clone_url":"https://github.com/other-org/repo.wiki.git","owner":{"login":"other-org","type":"Organization"},"size":1,"has_wiki":true}, + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{parentRepoURL, wikiSuffixedRepoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + installationID, mapped := connector.installationIDForRepo(parentRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) + + installationID, mapped = connector.installationIDForRepo(wikiSuffixedRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) + + installationID, mapped = connector.installationIDForRepo(wikiOfWikiSuffixedRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) +} + +func TestMapExplicitReposToInstallationsMapsRedirectedRepoFromMetadata(t *testing.T) { + privateKey := createPrivateKey() + const oldRepoURL = "https://github.com/old-org/repo.git" + const canonicalRepoURL = "https://github.com/new-org/repo.git" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"new-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"new-org/repo","clone_url":"https://github.com/new-org/repo.git","owner":{"login":"new-org","type":"Organization"},"size":1} + ]}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/old-org/repo"): + if !strings.Contains(r.Header.Get("Authorization"), "token-2448") { + http.Error(w, "wrong installation token", http.StatusForbidden) + return + } + _, _ = w.Write([]byte(`{"name":"repo","full_name":"new-org/repo","clone_url":"https://github.com/new-org/repo.git","owner":{"login":"new-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + Repositories: []string{oldRepoURL}, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + + installationID, mapped := connector.installationIDForRepo(oldRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) + installationID, mapped = connector.installationIDForRepo(canonicalRepoURL) + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) + info, ok := s.repoInfoCache.get(oldRepoURL) + require.True(t, ok) + assert.Equal(t, "new-org/repo", info.fullName) +} + +func TestRepoURLFromTargetMetadataErrorsOnRepositoryLinkMismatch(t *testing.T) { + s := &Source{} + _, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ + Repository: "https://github.com/other-org/repo.git", + Link: "https://github.com/different-org/repo/blob/abcd1234/path/to/file.go#L1", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not match link repository") +} + +func TestRepoURLFromTargetMetadataErrorsOnHostMismatch(t *testing.T) { + s := &Source{} + _, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ + Repository: "https://github.company/other-org/repo.git", + Link: "https://github.com/other-org/repo/blob/abcd1234/path/to/file.go#L1", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not match link repository") +} + +func TestRepoURLFromTargetMetadataAllowsBareRepositoryName(t *testing.T) { + s := &Source{} + repoURL, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ + Repository: "repo", + Link: "https://github.com/other-org/repo/blob/abcd1234/path/to/file.go#L1", + }) + require.NoError(t, err) + assert.Equal(t, "https://github.com/other-org/repo.git", repoURL) +} + +func TestRepoURLFromTargetMetadataAllowsWikiRepositoryWithWikiLink(t *testing.T) { + s := &Source{} + repoURL, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ + Repository: "https://github.com/other-org/repo.wiki.git", + Link: "https://github.com/other-org/repo/wiki/path/to/file.go", + }) + require.NoError(t, err) + assert.Equal(t, "https://github.com/other-org/repo.git", repoURL) +} + +func TestScanAllInstallationsMapsUnitBeforeMetadataFetch(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[ + {"id":1337,"account":{"login":"default-org","type":"Organization"}}, + {"id":2448,"account":{"login":"other-org","type":"Organization"}} + ]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + auth := r.Header.Get("Authorization") + switch { + case strings.Contains(auth, "token-1337"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[]}`)) + case strings.Contains(auth, "token-2448"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} + ]}`)) + default: + http.Error(w, "unexpected installation token", http.StatusUnauthorized) + } + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): + if !strings.Contains(r.Header.Get("Authorization"), "token-2448") { + http.Error(w, "wrong installation token", http.StatusForbidden) + return + } + _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/other-org/repo.git"}) + require.NoError(t, err) + + repoURL, err := s.ensureRepoInfoCache(context.Background(), "https://github.com/other-org/repo.git", &unitErrorReporter{noopReporter()}) + require.NoError(t, err) + require.Equal(t, "https://github.com/other-org/repo.git", repoURL) + + installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") + assert.True(t, mapped) + assert.Equal(t, int64(2448), installationID) +} + +// Regression test for member personal repos under scan_all_installations +// (INT-789): they belong to no app installation, so the mapping must fall +// back to the default installation when the API confirms the repo is +// readable, both at chunk time and at enumeration time. +func TestScanAllInstallationsMemberPersonalRepoFallsBackToDefaultInstallation(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[ + {"id":1337,"account":{"login":"test-org","type":"Organization"}}, + {"id":2448,"account":{"login":"other-org","type":"Organization"}} + ]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + // Installations only ever contain org-owned repos. + auth := r.Header.Get("Authorization") + switch { + case strings.Contains(auth, "token-1337"): + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ + {"name":"backend","full_name":"test-org/backend","clone_url":"https://github.com/test-org/backend.git","owner":{"login":"test-org","type":"Organization"},"size":1} + ]}`)) + case strings.Contains(auth, "token-2448"): + _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) + default: + http.Error(w, "unexpected installation token", http.StatusUnauthorized) + } + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/alice/dns-data"): + // The member's personal repo exists and is public: any valid + // token can read it, mirroring real GitHub behavior. + _, _ = w.Write([]byte(`{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/users/alice/repos"): + _, _ = w.Write([]byte(`[{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}]`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + ScanUsers: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + // The org-owned repo maps to its owning installation. + connector := s.connector.(*appConnector) + err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/test-org/backend.git"}) + require.NoError(t, err) + + // The member's public personal repo is in no installation, but the API + // confirms it is accessible, so it maps to the default installation. + err = s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/alice/dns-data.git"}) + require.NoError(t, err) + + installationID, mapped := connector.installationIDForRepo("https://github.com/alice/dns-data.git") + require.True(t, mapped) + require.Equal(t, int64(1337), installationID) + + // Enumeration must map member repos up front so same-process ChunkUnit + // short-circuits without any mapping API calls. + connector.mu.Lock() + delete(connector.repoInstallationMap, "https://github.com/alice/dns-data.git") + connector.mu.Unlock() + + require.NoError(t, s.getReposByUser(context.Background(), "alice", false, noopReporter())) + installationID, mapped = connector.installationIDForRepo("https://github.com/alice/dns-data.git") + require.True(t, mapped) + require.Equal(t, int64(1337), installationID) +} + +// Repos that no installation owns AND the default installation token cannot +// read must still fail the mapping: the accessibility fallback must not turn +// the check fail-open. +func TestScanAllInstallationsInaccessibleRepoStillFailsMapping(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"test-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) + default: + // /repos/... lookups 404: repo is private/nonexistent for + // every installation token. + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + connector := s.connector.(*appConnector) + err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/ghost/private-repo.git"}) + require.Error(t, err) + require.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation") +} + +func TestEnumerateAllInstallationReposReportsInstallationErrors(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}},{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + // Fail the first installation's repo listing; succeed for the second. + if strings.Contains(r.Header.Get("Authorization"), "token-1337") { + http.Error(w, "repo list failed", http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(`{"total_count":1,"repositories":[{"id":1,"name":"repo-b","full_name":"other-org/repo-b","clone_url":"https://github.com/other-org/repo-b.git","size":1}]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + var reportedErrs []error + var reportedUnits []sources.SourceUnit + reporter := sources.VisitorReporter{ + VisitUnit: func(_ context.Context, unit sources.SourceUnit) error { + reportedUnits = append(reportedUnits, unit) + return nil + }, + VisitErr: func(_ context.Context, err error) error { + reportedErrs = append(reportedErrs, err) + return nil + }, + } + + err := s.enumerateAllInstallationRepos(context.Background(), s.connector.(*appConnector), reporter) + + // A single installation failure is reported but doesn't abort the scan; + // the other installation's repos are still enumerated. + require.NoError(t, err) + require.Len(t, reportedErrs, 1) + assert.Contains(t, reportedErrs[0].Error(), "error enumerating repos for installation 1337") + require.Len(t, reportedUnits, 1) + unitID, _ := reportedUnits[0].SourceUnitID() + assert.Equal(t, "https://github.com/other-org/repo-b.git", unitID) +} + +func TestChunksReturnsScanAllInstallationEnumerationErrors(t *testing.T) { + privateKey := createPrivateKey() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): + parts := strings.Split(r.URL.Path, "/") + installID := parts[len(parts)-2] + _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): + _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}},{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): + if strings.Contains(r.Header.Get("Authorization"), "token-1337") { + http.Error(w, "repo list failed", http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + s, conn := createTestSource(&sourcespb.GitHub{ + Endpoint: server.URL, + ScanAllInstallations: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }, + }) + require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) + + chunks := make(chan *sources.Chunk, 1) + err := s.Chunks(context.Background(), chunks) + + require.Error(t, err) + assert.Contains(t, err.Error(), "error enumerating repos for installation 1337") +} + +// This only tests the resume info slice portion of setProgressCompleteWithRepo. +func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) { + tests := []struct { + startingResumeInfoSlice []string + repoURL string + wantResumeInfoSlice []string + }{ + { + startingResumeInfoSlice: []string{}, + repoURL: "a", + wantResumeInfoSlice: []string{"a"}, + }, + { + startingResumeInfoSlice: []string{"b"}, + repoURL: "a", + wantResumeInfoSlice: []string{"a", "b"}, + }, + } + + s := &Source{ + repos: []string{}, + } + + for _, tt := range tests { + s.resumeInfoSlice = tt.startingResumeInfoSlice + s.setProgressCompleteWithRepo(0, 0, tt.repoURL) + if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) { + t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice) + } + } +} + +func Test_setProgressCompleteWithRepo_Progress(t *testing.T) { + repos := []string{"a", "b", "c", "d", "e"} + tests := map[string]struct { + repos []string + index int + offset int + wantPercentComplete int64 + wantSectionsCompleted int32 + wantSectionsRemaining int32 + }{ + "starting from the beginning, no offset": { + repos: repos, + index: 0, + offset: 0, + wantPercentComplete: 0, + wantSectionsCompleted: 0, + wantSectionsRemaining: 5, + }, + "resume from the third, offset 2": { + repos: repos[2:], + index: 0, + offset: 2, + wantPercentComplete: 40, + wantSectionsCompleted: 2, + wantSectionsRemaining: 5, + }, + "resume from the third, on last repo, offset 2": { + repos: repos[2:], + index: 2, + offset: 2, + wantPercentComplete: 80, + wantSectionsCompleted: 4, + wantSectionsRemaining: 5, + }, + } + + for _, tt := range tests { + s := &Source{ + repos: tt.repos, + } + + s.setProgressCompleteWithRepo(tt.index, tt.offset, "") + gotProgress := s.GetProgress() + if gotProgress.PercentComplete != tt.wantPercentComplete { + t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete) + } + if gotProgress.SectionsCompleted != tt.wantSectionsCompleted { + t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted) + } + if gotProgress.SectionsRemaining != tt.wantSectionsRemaining { + t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining) + } + } +} + +func Test_scan_SetProgressComplete(t *testing.T) { + testCases := []struct { + name string + repos []string + wantComplete bool + wantErr bool + }{ + { + name: "no repos", + wantComplete: true, + }, + { + name: "one valid repo", + repos: []string{"https://github.com/super-secret-user/super-secret-repo.git"}, + wantComplete: true, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + src := initTestSource(&sourcespb.GitHub{ + Repositories: tc.repos, + Credential: &sourcespb.GitHub_Unauthenticated{}, + }) + src.jobPool = &errgroup.Group{} + + _ = src.scan(context.Background(), nil) + if !tc.wantErr { + assert.Equal(t, "", src.GetProgress().EncodedResumeInfo) + } + + gotComplete := src.GetProgress().PercentComplete == 100 + if gotComplete != tc.wantComplete { + t.Errorf("got: %v, want: %v", gotComplete, tc.wantComplete) + } + }) + } +} + +func TestGetRepoURLParts(t *testing.T) { + repoURLs := []string{ + "https://github.com/trufflesecurity/trufflehog.git", + "git+https://github.com/trufflesecurity/trufflehog.git", + "ssh://github.com/trufflesecurity/trufflehog.git", + "ssh://git@github.com/trufflesecurity/trufflehog.git", + "git+ssh://git@github.com/trufflesecurity/trufflehog.git", + "git://github.com/trufflesecurity/trufflehog.git", + } + expected := []string{"github.com", "trufflesecurity", "trufflehog"} + for _, tt := range repoURLs { + _, parts, err := getRepoURLParts(tt) + if err != nil { + t.Fatalf("failed: %v", err) + } + assert.Equal(t, expected, parts) + } + + gistURLs := map[string][]string{ + // Gists + "ssh://github.com/6df198861306313246466d23aa4102aa.git": nil, + "ssh://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"}, + "https://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"}, + "https://gist.github.com/john-smith/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "john-smith", "6df198861306313246466d23aa4102aa"}, + "ssh://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"}, + "https://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"}, + "https://github.contoso.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "john-smith", "6df198861306313246466d23aa4102aa"}, + "https://github.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": nil, + } + for tt, expected := range gistURLs { + _, parts, err := getRepoURLParts(tt) + if err != nil { + if expected == nil { + continue + } + t.Fatalf("failed: %v", err) + } + assert.Equal(t, expected, parts) + } +} + +func TestGetRepoURLPartsWithTrailingHyphen(t *testing.T) { + // Test for https://github.com/trufflesecurity/trufflehog/issues/4679 + // Repository names ending with a hyphen should be preserved correctly. + testCases := []struct { + name string url string expected []string }{ @@ -1105,6 +2219,74 @@ func TestGetRepoURLPartsWithTrailingHyphen(t *testing.T) { } } +func TestGithubWebHost(t *testing.T) { + tests := []struct { + name string + endpoint string + want string + }{ + { + name: "empty endpoint defaults to github", + endpoint: "", + want: "github.com", + }, + { + name: "cloud api endpoint", + endpoint: "https://api.github.com", + want: "github.com", + }, + { + name: "cloud api endpoint without scheme", + endpoint: "api.github.com", + want: "github.com", + }, + { + name: "enterprise endpoint", + endpoint: "https://github.example.com/api/v3", + want: "github.example.com", + }, + { + name: "enterprise endpoint without scheme", + endpoint: "github.example.com/api/v3", + want: "github.example.com", + }, + { + name: "host ending with github.com substring is enterprise", + endpoint: "https://legithub.com/api/v3", + want: "legithub.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, githubWebHost(tt.endpoint)) + }) + } +} + +func TestWikiIsReachableSkipsHeadForPrivateRepos(t *testing.T) { + var called bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + http.Redirect(w, r, "/login", http.StatusFound) + })) + defer server.Close() + + repoURL := "https://github.example.com/owner/repo.git" + s := &Source{ + conn: &sourcespb.GitHub{Endpoint: server.URL}, + repoInfoCache: newRepoInfoCache(), + } + s.repoInfoCache.put(repoURL, repoInfo{ + owner: "owner", + name: "repo", + visibility: source_metadatapb.Visibility_private, + }) + + assert.True(t, s.wikiIsReachable(context.Background(), repoURL)) + assert.False(t, called) +} + func TestGetGistID(t *testing.T) { tests := []struct { trimmedURL []string @@ -1146,6 +2328,62 @@ func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) { } } +func TestScanAllInstallationsTargetMappingErrorIsTargetedScanError(t *testing.T) { + s := &Source{ + conn: &sourcespb.GitHub{ScanAllInstallations: true}, + connector: &appConnector{ + clientsByInstallationID: make(map[int64]*appInstallationClients), + repoInstallationMap: make(map[string]int64), + }, + } + ctx := context.Background() + chunksChan := make(chan *sources.Chunk) + + targets := []sources.ChunkingTarget{ + { + SecretID: 1, + QueryCriteria: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{Link: "https://github.com/only-owner"}, + }, + }, + }, + {SecretID: 2}, + } + + err := s.Chunks(ctx, chunksChan, targets...) + unwrappable, ok := err.(interface{ Unwrap() []error }) + if assert.True(t, ok, "returned error was not unwrappable") { + got := unwrappable.Unwrap() + require.Len(t, got, 2) + for _, targetErr := range got { + assert.IsType(t, &sources.TargetedScanError{}, targetErr) + } + } +} + +func TestScanTargetRejectsWikiLinkHostMismatch(t *testing.T) { + s := &Source{ + conn: &sourcespb.GitHub{Endpoint: cloudV3Endpoint}, + } + target := sources.ChunkingTarget{ + QueryCriteria: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Link: "https://attacker.example/owner/repo/wiki/path/to/file", + Commit: "abc123", + File: "path/to/file", + }, + }, + }, + } + + err := s.scanTarget(context.Background(), target, sources.ChanReporter{Ch: make(chan *sources.Chunk, 1)}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "does not match GitHub endpoint host") +} + func TestRepositoryFiltering(t *testing.T) { // Test that the filteredRepoCache correctly filters repositories source := &Source{} @@ -1228,6 +2466,104 @@ func TestExplicitRepositoryBypass(t *testing.T) { assert.Contains(t, source.repos, "https://github.com/org/another-explicit.git") } +// TestEnsureRepoInfoCache_RepoRedirect verifies that when the GitHub API returns +// a canonical CloneURL that differs from the requested URL (i.e. a repository +// has been renamed/moved), ensureRepoInfoCache stores info under BOTH the +// original URL and the canonical URL so that scanRepo and visibilityOf can +// look up metadata using either. +func TestEnsureRepoInfoCache_RepoRedirect(t *testing.T) { + defer gock.Off() + + const ( + originalURL = "https://github.com/old-org/myrepo.git" + canonicalURL = "https://github.com/new-org/myrepo.git" + ) + + gock.New("https://api.github.com"). + Get("/repos/old-org/myrepo"). + Reply(200). + JSON(map[string]any{ + "name": "myrepo", + "full_name": "new-org/myrepo", + "clone_url": canonicalURL, + "private": false, + "has_wiki": false, + "size": 100, + "owner": map[string]string{"login": "new-org"}, + }) + + s := initTestSource(&sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Token{Token: "token"}, + }) + + returnedURL, err := s.ensureRepoInfoCache(context.Background(), originalURL, &unitErrorReporter{noopReporter()}) + require.NoError(t, err) + + // Original URL returned so callers can continue using it. + assert.Equal(t, originalURL, returnedURL) + + // Both URLs must be in the cache so scanRepo / visibilityOf don't miss. + _, okOriginal := s.repoInfoCache.get(originalURL) + assert.True(t, okOriginal, "original (pre-redirect) URL must be cached") + + _, okCanonical := s.repoInfoCache.get(canonicalURL) + assert.True(t, okCanonical, "canonical (post-redirect) URL must be cached") + + assert.False(t, gock.HasUnmatchedRequest()) + assert.True(t, gock.IsDone()) +} + +// TestEnumerate_ExplicitRepoRedirect verifies the end-to-end Enumerate flow +// when a user provides an explicit repo URL that GitHub redirects to a new +// location (e.g. org rename). scanRepo must be able to look up cache metadata +// using the original URL that ends up in s.repos. +func TestEnumerate_ExplicitRepoRedirect(t *testing.T) { + defer gock.Off() + + const ( + originalURL = "https://github.com/old-org/myrepo.git" + canonicalURL = "https://github.com/new-org/myrepo.git" + ) + + gock.New("https://api.github.com"). + Get("/user"). + Reply(200). + JSON(map[string]string{"login": "test-user"}) + + gock.New("https://api.github.com"). + Get("/repos/old-org/myrepo"). + Reply(200). + JSON(map[string]any{ + "name": "myrepo", + "full_name": "new-org/myrepo", + "clone_url": canonicalURL, + "private": true, + "has_wiki": false, + "size": 50, + "owner": map[string]string{"login": "new-org"}, + }) + + s := initTestSource(&sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Token{Token: "token"}, + Repositories: []string{originalURL}, + }) + + err := s.Enumerate(context.Background(), noopReporter()) + require.NoError(t, err) + + // s.repos must use the original URL (what was configured). + assert.Contains(t, s.repos, originalURL) + + // scanRepo looks up repoInfoCache by the URL in s.repos. + // Without the redirect fix this lookup fails with "no repoInfo for URL". + info, ok := s.repoInfoCache.get(originalURL) + assert.True(t, ok, "scanRepo must be able to look up metadata by original URL") + assert.Equal(t, "new-org", info.owner) + + assert.False(t, gock.HasUnmatchedRequest()) + assert.True(t, gock.IsDone()) +} + func noopReporter() sources.UnitReporter { return sources.VisitorReporter{ VisitUnit: func(context.Context, sources.SourceUnit) error { @@ -1310,3 +2646,349 @@ func TestExtractRepoNameFromURL(t *testing.T) { }) } } + +func TestSource_ExcludeArchivedRepositories(t *testing.T) { + tests := []struct { + name string + excludeArchived bool + reposJSON string + wantRepoCount int + wantRepos []string + }{ + { + name: "exclude archived repos when flag is true", + excludeArchived: true, + reposJSON: `[ + {"full_name": "test-org/active-repo", "clone_url": "https://github.com/test-org/active-repo.git", "size": 1, "archived": false}, + {"full_name": "test-org/archived-repo", "clone_url": "https://github.com/test-org/archived-repo.git", "size": 1, "archived": true}, + {"full_name": "test-org/another-active", "clone_url": "https://github.com/test-org/another-active.git", "size": 1, "archived": false} + ]`, + wantRepoCount: 2, // Only non-archived + wantRepos: []string{"test-org/active-repo", "test-org/another-active"}, + }, + { + name: "include archived repos when flag is false", + excludeArchived: false, + reposJSON: `[ + {"full_name": "test-org/active-repo", "clone_url": "https://github.com/test-org/active-repo.git", "size": 1, "archived": false}, + {"full_name": "test-org/archived-repo", "clone_url": "https://github.com/test-org/archived-repo.git", "size": 1, "archived": true}, + {"full_name": "test-org/another-active", "clone_url": "https://github.com/test-org/another-active.git", "size": 1, "archived": false} + ]`, + wantRepoCount: 3, // All repos + wantRepos: []string{"test-org/active-repo", "test-org/archived-repo", "test-org/another-active"}, + }, + { + name: "handle all archived repos", + excludeArchived: true, + reposJSON: `[ + {"full_name": "test-org/archived-1", "clone_url": "https://github.com/test-org/archived-1.git", "size": 1, "archived": true}, + {"full_name": "test-org/archived-2", "clone_url": "https://github.com/test-org/archived-2.git", "size": 1, "archived": true} + ]`, + wantRepoCount: 0, // None included + wantRepos: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/orgs/test-org/repos"). + Reply(200). + JSON(tt.reposJSON) + + s := initTestSource(&sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Token{ + Token: "test-token", + }, + ExcludeArchived: tt.excludeArchived, + }) + + err := s.getReposByOrg(context.Background(), "test-org", noopReporter()) + assert.Nil(t, err) + assert.Equal(t, tt.wantRepoCount, s.filteredRepoCache.Count()) + + // Verify expected repos are in cache + for _, repo := range tt.wantRepos { + ok := s.filteredRepoCache.Exists(repo) + assert.True(t, ok, "expected repo %s to be in cache", repo) + } + + // Verify archived repos are NOT in cache when excluding + if tt.excludeArchived { + allRepos := []string{"test-org/archived-repo", "test-org/archived-1", "test-org/archived-2"} + for _, repo := range allRepos { + if !slices.Contains(tt.wantRepos, repo) { + ok := s.filteredRepoCache.Exists(repo) + assert.False(t, ok, "archived repo %s should not be in cache when excluding", repo) + } + } + } + + assert.False(t, gock.HasUnmatchedRequest()) + assert.True(t, gock.IsDone()) + }) + } +} + +func TestSource_ExcludeArchivedForkInteraction(t *testing.T) { + tests := []struct { + name string + includeForks bool + excludeArchived bool + reposJSON string + wantRepoCount int + wantRepos []string + notWantRepos []string + }{ + { + name: "fork+archived skipped when forks included and archived excluded", + includeForks: true, + excludeArchived: true, + reposJSON: `[ + {"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false}, + {"full_name": "test-org/fork-active", "clone_url": "https://github.com/test-org/fork-active.git", "size": 1, "archived": false, "fork": true}, + {"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true} + ]`, + wantRepoCount: 2, + wantRepos: []string{"test-org/active", "test-org/fork-active"}, + notWantRepos: []string{"test-org/fork-archived"}, + }, + { + name: "fork+archived included when archived allowed", + includeForks: true, + excludeArchived: false, + reposJSON: `[ + {"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false}, + {"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true} + ]`, + wantRepoCount: 2, + wantRepos: []string{"test-org/active", "test-org/fork-archived"}, + }, + { + name: "fork+archived dropped by fork filter when forks excluded", + includeForks: false, + excludeArchived: true, + reposJSON: `[ + {"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false}, + {"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true} + ]`, + wantRepoCount: 1, + wantRepos: []string{"test-org/active"}, + notWantRepos: []string{"test-org/fork-archived"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/orgs/test-org/repos"). + Reply(200). + JSON(tt.reposJSON) + + s := initTestSource(&sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Token{ + Token: "test-token", + }, + IncludeForks: tt.includeForks, + ExcludeArchived: tt.excludeArchived, + }) + + err := s.getReposByOrg(context.Background(), "test-org", noopReporter()) + assert.Nil(t, err) + assert.Equal(t, tt.wantRepoCount, s.filteredRepoCache.Count()) + + for _, repo := range tt.wantRepos { + assert.True(t, s.filteredRepoCache.Exists(repo), "expected repo %s to be in cache", repo) + } + for _, repo := range tt.notWantRepos { + assert.False(t, s.filteredRepoCache.Exists(repo), "repo %s should not be in cache", repo) + } + + assert.False(t, gock.HasUnmatchedRequest()) + assert.True(t, gock.IsDone()) + }) + } +} + +func TestIsGHECloud(t *testing.T) { + tests := []struct { + name string + endpoint string + want bool + }{ + // GHE.com endpoints — should return true + {"api subdomain", "https://api.company.ghe.com", true}, + {"web subdomain", "https://company.ghe.com", true}, + {"api subdomain trailing slash", "https://api.company.ghe.com/", true}, + {"web subdomain trailing slash", "https://company.ghe.com/", true}, + {"uppercase", "https://API.company.GHE.COM", true}, + {"mixed case", "https://Api.company.Ghe.Com/", true}, + {"deep subdomain", "https://api.team.company.ghe.com", true}, + + // GHES endpoints — should return false + {"ghes custom domain", "https://github.mycompany.com", false}, + {"ghes with path", "https://github.mycompany.com/api/v3", false}, + {"ghes internal", "https://github.internal.corp.com", false}, + + // github.com — should return false + {"cloud api", "https://api.github.com", false}, + {"cloud web", "https://github.com", false}, + + // Edge cases — should return false + {"empty string", "", false}, + {"localhost", "http://localhost:8080", false}, + {"malicious suffix", "https://notghe.com", false}, + {"partial match", "https://api.fakeghe.com.evil.com", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isGHECloud(tt.endpoint) + if got != tt.want { + t.Errorf("isGHECloud(%q) = %v, want %v", tt.endpoint, got, tt.want) + } + }) + } +} + +func TestNormalizeGHECloudAPIEndpoint(t *testing.T) { + tests := []struct { + name string + input string + want string + wantErr bool + }{ + { + name: "web URL to API URL", + input: "https://company.ghe.com", + want: "https://api.company.ghe.com/", + }, + { + name: "web URL with trailing slash", + input: "https://company.ghe.com/", + want: "https://api.company.ghe.com/", + }, + { + name: "api URL already correct", + input: "https://api.company.ghe.com", + want: "https://api.company.ghe.com/", + }, + { + name: "api URL with trailing slash", + input: "https://api.company.ghe.com/", + want: "https://api.company.ghe.com/", + }, + { + name: "preserves port", + input: "http://company.ghe.com:8080", + want: "http://api.company.ghe.com:8080/", + }, + { + name: "api with port", + input: "http://api.company.ghe.com:8080", + want: "http://api.company.ghe.com:8080/", + }, + { + name: "strips path", + input: "https://company.ghe.com/some/path", + want: "https://api.company.ghe.com/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeGHECloudAPIEndpoint(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("normalizeGHECloudAPIEndpoint(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("normalizeGHECloudAPIEndpoint(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestCreateAPIClient_GHECloud_NoAPIV3(t *testing.T) { + // The most critical test: GHE.com clients must NOT have /api/v3/ in their BaseURL. + endpoints := []string{ + "https://api.company.ghe.com", + "https://company.ghe.com", + "https://api.company.ghe.com/", + "https://company.ghe.com/", + } + + for _, endpoint := range endpoints { + t.Run(endpoint, func(t *testing.T) { + client, err := createGHECloudClient(http.DefaultClient, endpoint) + if err != nil { + t.Fatalf("createGHECloudClient(%q) unexpected error: %v", endpoint, err) + } + + baseURL := client.BaseURL.String() + + // Must NOT contain /api/v3 + if strings.Contains(baseURL, "/api/v3") { + t.Errorf("GHE.com client BaseURL must not contain /api/v3/, got: %s", baseURL) + } + + // Must point to api.company.ghe.com + if client.BaseURL.Hostname() != "api.company.ghe.com" { + t.Errorf("expected hostname api.company.ghe.com, got: %s", client.BaseURL.Hostname()) + } + + // Must have trailing slash + if !strings.HasSuffix(baseURL, "/") { + t.Errorf("BaseURL must have trailing slash, got: %s", baseURL) + } + }) + } +} + +func TestCreateGraphqlClient_GHECloud(t *testing.T) { + // GHE.com GraphQL should be at /graphql, NOT /api/graphql. + // We can't easily inspect the URL from the githubv4.Client, + // so we test the URL construction logic indirectly by checking + // the normalized endpoint. + apiURL, err := normalizeGHECloudAPIEndpoint("https://company.ghe.com") + if err != nil { + t.Fatal(err) + } + + graphqlURL := strings.TrimRight(apiURL, "/") + "/graphql" + expected := "https://api.company.ghe.com/graphql" + + if graphqlURL != expected { + t.Errorf("GHE.com GraphQL URL = %q, want %q", graphqlURL, expected) + } +} + +func TestCreateAPIClient_GHES_HasAPIV3(t *testing.T) { + // GHES clients SHOULD have /api/v3/ — make sure we didn't break that. + // Note: WithEnterpriseURLs returns an error for invalid URLs, but for + // valid ones it appends /api/v3/. + client, err := createAPIClient(context.Background(), http.DefaultClient, "https://github.mycompany.com") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !strings.Contains(client.BaseURL.String(), "/api/v3") { + t.Errorf("GHES client BaseURL should contain /api/v3/, got: %s", client.BaseURL.String()) + } +} + +func TestCreateAPIClient_CloudGitHub(t *testing.T) { + // Regular github.com should use the default client (api.github.com). + client, err := createAPIClient(context.Background(), http.DefaultClient, cloudV3Endpoint) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if client.BaseURL.String() != "https://api.github.com/" { + t.Errorf("expected https://api.github.com/, got: %s", client.BaseURL.String()) + } +} diff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.go index b82a93fe1dd4..4efb21c17444 100644 --- a/pkg/sources/github/graphql.go +++ b/pkg/sources/github/graphql.go @@ -17,7 +17,7 @@ import ( // processIssuesWithComments process github repo issues with comments using graphql API func (s *Source) processIssuesWithComments( - ctx context.Context, repoInfo repoInfo, + ctx context.Context, graphqlClient *githubv4.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time, ) error { vars := map[string]any{ @@ -34,7 +34,7 @@ func (s *Source) processIssuesWithComments( // loop will continue as long as there are issues in the repository for { var query issuesWithComments - err := s.connector.GraphQLClient().Query(ctx, &query, vars) + err := graphqlClient.Query(ctx, &query, vars) if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { continue } @@ -75,7 +75,7 @@ func (s *Source) processIssuesWithComments( // request this issue more comments var commentsQuery singleIssueComments - err := s.connector.GraphQLClient().Query(ctx, &commentsQuery, commentVars) + err := graphqlClient.Query(ctx, &commentsQuery, commentVars) if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentsQuery.RateLimit, err) { continue } @@ -111,7 +111,7 @@ func (s *Source) processIssuesWithComments( } // processPRWithComments process github repo pull requests with inline comments -func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githubv4.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { vars := map[string]any{ owner: githubv4.String(repoInfo.owner), repository: githubv4.String(repoInfo.name), @@ -126,7 +126,7 @@ func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, r // continue loop as long as there are pull requests remaining for { var query pullRequestWithComments - err := s.connector.GraphQLClient().Query(ctx, &query, vars) + err := graphqlClient.Query(ctx, &query, vars) if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { continue } @@ -166,7 +166,7 @@ func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, r commentsPagination: pr.Comments.PageInfo.EndCursor, } - err := s.connector.GraphQLClient().Query(ctx, &commentQuery, singlePRVars) + err := graphqlClient.Query(ctx, &commentQuery, singlePRVars) if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentQuery.RateLimit, err) { continue } @@ -202,7 +202,7 @@ func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, r } // processReviewThreads process github repo pull request review threads -func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) processReviewThreads(ctx context.Context, graphqlClient *githubv4.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { vars := map[string]any{ owner: githubv4.String(repoInfo.owner), repository: githubv4.String(repoInfo.name), @@ -217,7 +217,7 @@ func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, re // continue as long as pull requests have threads for { var query prWithReviewThreadIDs - err := s.connector.GraphQLClient().Query(ctx, &query, vars) + err := graphqlClient.Query(ctx, &query, vars) if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { continue } @@ -249,7 +249,7 @@ func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, re for _, batch := range chunkIDs(threadIDs, 100) { ctx.Logger().V(5).Info("Processing Thread comments in Batches", "batch_length", len(batch)) // fetch comments for the batch of threads - if err := s.fetchThreadComments(ctx, batch, repoInfo, reporter, cutoffTime); err != nil { + if err := s.fetchThreadComments(ctx, graphqlClient, batch, repoInfo, reporter, cutoffTime); err != nil { return fmt.Errorf("error fetching thread review comments: %w", err) } } @@ -259,7 +259,7 @@ func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, re } // fetchThreadComments process github repo pull request threads and their comments -func (s *Source) fetchThreadComments(ctx context.Context, threadIDs []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { +func (s *Source) fetchThreadComments(ctx context.Context, graphqlClient *githubv4.Client, threadIDs []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error { // Process in batches of 100 var query multiReviewThreadComments vars := map[string]any{ @@ -268,7 +268,7 @@ func (s *Source) fetchThreadComments(ctx context.Context, threadIDs []string, re commentsPagination: (*githubv4.String)(nil), } - if err := s.connector.GraphQLClient().Query(ctx, &query, vars); err != nil { + if err := graphqlClient.Query(ctx, &query, vars); err != nil { return fmt.Errorf("multi-thread query failed: %w", err) } @@ -288,7 +288,7 @@ func (s *Source) fetchThreadComments(ctx context.Context, threadIDs []string, re commentsPagination: (*githubv4.String)(nil), } - err := s.connector.GraphQLClient().Query(ctx, &query, reviewThreadVars) + err := graphqlClient.Query(ctx, &query, reviewThreadVars) if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) { continue } @@ -348,8 +348,8 @@ func (s *Source) handleGraphQLRateLimit(ctx context.Context, rl *rateLimit, errI // if resume time is not empty and is after current time, than put the request to sleep till that. if !resumeTime.IsZero() && time.Now().Before(resumeTime) { retryAfter := time.Until(resumeTime) - time.Sleep(retryAfter) - return true + _, canceled := sleepWithContext(ctx, retryAfter) + return !canceled } var retryAfter time.Duration @@ -393,10 +393,10 @@ func (s *Source) handleGraphQLRateLimit(ctx context.Context, rl *rateLimit, errI } githubNumRateLimitEncountered.WithLabelValues(s.name).Inc() - time.Sleep(retryAfter) - githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds()) + slept, canceled := sleepWithContext(ctx, retryAfter) + githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(slept.Seconds()) - return true + return !canceled } func (s *Source) chunkGraphqlIssues(ctx context.Context, repoInfo repoInfo, issues []issue, reporter sources.ChunkReporter) error { diff --git a/pkg/sources/github/map_installations_test.go b/pkg/sources/github/map_installations_test.go new file mode 100644 index 000000000000..51c4fda5173d --- /dev/null +++ b/pkg/sources/github/map_installations_test.go @@ -0,0 +1,167 @@ +package github + +import ( + "encoding/json" + "testing" + + "github.com/google/go-github/v67/github" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// This is meant to track APIUnitReporter.UnitOK in scanner/cmd/scanner/pipeline.go +// precisely +func newMarshalledRepoUnit(t *testing.T, su sources.SourceUnit) []byte { + t.Helper() + + id, kind := su.SourceUnitID() + + unitData, err := json.Marshal(su) + require.NoError(t, err) + + out, err := json.Marshal(map[string]any{ + "id": id, + "kind": string(kind), + "display": su.Display(), + "unit_data": unitData, + }) + require.NoError(t, err) + + return out +} + +func TestGitHub_UnmarshalSourceUnit(t *testing.T) { + s := &Source{} + + t.Run("source unit with unit_data recovers installation id", func(t *testing.T) { + data := newMarshalledRepoUnit(t, RepoUnit{ + Name: "spotify/backstage", + URL: "https://github.com/spotify/backstage.git", + InstallationID: 2448, + }) + + unit, err := s.UnmarshalSourceUnit(data) + require.NoError(t, err) + + repoUnit, ok := unit.(RepoUnit) + require.True(t, ok, "expected a RepoUnit so ChunkUnit can read the installation id") + assert.Equal(t, "https://github.com/spotify/backstage.git", repoUnit.URL) + assert.Equal(t, "spotify/backstage", repoUnit.Name) + assert.Equal(t, int64(2448), repoUnit.InstallationID) + + id, kind := unit.SourceUnitID() + assert.Equal(t, "https://github.com/spotify/backstage.git", id) + assert.EqualValues(t, "repo", kind) + }) + + t.Run("legacy source unit without unit_data has no installation id", func(t *testing.T) { + data, err := json.Marshal(map[string]any{ + "id": "https://github.com/acme/widgets.git", + "kind": "repo", + "display": "acme/widgets", + }) + require.NoError(t, err) + + unit, err := s.UnmarshalSourceUnit(data) + require.NoError(t, err) + + repoUnit, ok := unit.(RepoUnit) + require.True(t, ok) + assert.Equal(t, "https://github.com/acme/widgets.git", repoUnit.URL) + assert.Equal(t, "acme/widgets", repoUnit.Name) + assert.Zero(t, repoUnit.InstallationID, "legacy units fall back to deriving the installation") + }) + + t.Run("full enumeration payload", func(t *testing.T) { + data, err := json.Marshal(RepoUnit{ + Name: "acme/widgets", + URL: "https://github.com/acme/widgets.git", + InstallationID: 111, + }) + require.NoError(t, err) + + unit, err := s.UnmarshalSourceUnit(data) + require.NoError(t, err) + + repoUnit, ok := unit.(RepoUnit) + require.True(t, ok) + assert.Equal(t, int64(111), repoUnit.InstallationID) + }) + + t.Run("gist unit round trips", func(t *testing.T) { + data := newMarshalledRepoUnit(t, GistUnit{Name: "abc", URL: "https://gist.github.com/abc.git"}) + + unit, err := s.UnmarshalSourceUnit(data) + require.NoError(t, err) + + _, ok := unit.(GistUnit) + require.True(t, ok) + _, kind := unit.SourceUnitID() + assert.EqualValues(t, "gist", kind) + }) + + t.Run("rejects unrecognized payload", func(t *testing.T) { + _, err := s.UnmarshalSourceUnit([]byte(`{"unrelated":"value"}`)) + require.Error(t, err) + }) +} + +// Ensure we store installation IDs when enumerating +func TestGitHub_ProcessReposEmitsInstallationID(t *testing.T) { + ctx := context.Background() + + connector := &appConnector{ + installationID: 1, + repoInstallationMap: make(map[string]int64), + } + s := &Source{ + conn: &sourcespb.GitHub{ScanAllInstallations: true}, + connector: connector, + filteredRepoCache: &filteredRepoCache{Cache: simple.NewCache[string]()}, + repoInfoCache: newRepoInfoCache(), + } + + const installID int64 = 2448 + repo := &github.Repository{ + Name: github.String("backstage"), + FullName: github.String("spotify/backstage"), + CloneURL: github.String("https://github.com/spotify/backstage.git"), + Owner: &github.User{Login: github.String("spotify"), Type: github.String("Organization")}, + } + + // Mirror enumerateAllInstallationRepos: the lister records the installation + // for each repo in a page before processRepos emits the units for it. + page := 0 + listRepos := func(ctx context.Context, _ string, _ repoListOptions) ([]*github.Repository, *github.Response, error) { + page++ + if page > 1 { + return nil, &github.Response{}, nil + } + connector.setRepoInstallationForRepoName(repo.GetCloneURL(), repo.GetName(), installID) + return []*github.Repository{repo}, &github.Response{}, nil + } + + var units []sources.SourceUnit + reporter := sources.VisitorReporter{ + VisitUnit: func(_ context.Context, u sources.SourceUnit) error { + units = append(units, u) + return nil + }, + } + + err := s.processRepos(ctx, "spotify", reporter, listRepos, &appListOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + }) + require.NoError(t, err) + + require.Len(t, units, 1) + repoUnit, ok := units[0].(RepoUnit) + require.True(t, ok) + assert.Equal(t, "https://github.com/spotify/backstage.git", repoUnit.URL) + assert.Equal(t, installID, repoUnit.InstallationID) +} diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index ad20fc79432f..a74698a09343 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -6,17 +6,18 @@ import ( "io" "net/http" "net/url" - "regexp" "strings" "sync" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v67/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + sourcegit "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) // repoInfoCache is a thread-safe cache to store information about repositories. @@ -54,9 +55,9 @@ type repoInfo struct { owner string // repository owner (user|organization). name string // repository name. fullName string // full repository name (owner/repo). - hasWiki bool // whether the repository is likely to have a wiki. size int // size of the repository in kilobytes. visibility source_metadatapb.Visibility // visibility of the repository (public/private). + hasWiki bool // whether the repository is likely to have a wiki. } // cloneRepo clones a repository given its URL, returns the path and the repository object. @@ -245,8 +246,9 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc opts := listOpts.getListOptions() var ( - numRepos, numForks int - uniqueOrgs = map[string]struct{}{} + numRepos, numForks, numArchived int + numArchivedSkipped int + uniqueOrgs = map[string]struct{}{} ) // loop to handle pagination. @@ -261,12 +263,26 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc ctx.Logger().V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage) for _, r := range someRepos { - if r.GetFork() { - if !s.conn.IncludeForks { - continue - } + isFork, isArchived := r.GetFork(), r.GetArchived() + + if isFork && !s.conn.IncludeForks { + continue + } + + if isArchived && s.conn.ExcludeArchived { + numArchivedSkipped++ + logger.V(3).Info("skipping archived repository", "repo", r.GetFullName()) + continue + } + + if isFork { numForks++ } + + if isArchived { + numArchived++ + } + numRepos++ // track unique organizations. @@ -284,7 +300,22 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc s.totalRepoSize += r.GetSize() s.filteredRepoCache.Set(repoName, repoURL) s.cacheRepoInfo(r) - if err := reporter.UnitOk(ctx, RepoUnit{Name: repoName, URL: repoURL}); err != nil { + + unit := RepoUnit{Name: repoName, URL: repoURL} + // Repos enumerated outside installation listings (e.g. member + // personal repos) belong to no installation; map them to the + // default installation so the scan-all mapping in ChunkUnit + // doesn't reject repos we just enumerated. + if connector, ok := s.connector.(*appConnector); ok && s.conn.GetScanAllInstallations() { + connector.ensureRepoInstallation(repoURL, r.GetName()) + // Record the resolved installation on the unit so ChunkUnit + // scans it with the correct token without re-listing every + // installation's repos (INT-790). + if installID, mapped := connector.installationIDForRepo(repoURL); mapped { + unit.InstallationID = installID + } + } + if err := reporter.UnitOk(ctx, unit); err != nil { return err } @@ -298,7 +329,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc } // final logging of repository stats. - logger.V(2).Info("found repos", "total", numRepos, "num_forks", numForks, "num_orgs", len(uniqueOrgs)) + logger.V(2).Info("found repos", "total", numRepos, "num_forks", numForks, "num_archived", numArchived, "num_archived_skipped", numArchivedSkipped, "num_orgs", len(uniqueOrgs)) githubOrgsEnumerated.WithLabelValues(s.name).Add(float64(len(uniqueOrgs))) return nil @@ -338,13 +369,25 @@ func (s *Source) cacheGistInfo(g *github.Gist) { // Unfortunately, this isn't 100% accurate. Some repositories have `has_wiki: true` and don't redirect their wiki page, // but still don't have a cloneable wiki. func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool { - wikiURL := strings.TrimSuffix(repoURL, ".git") + "/wiki" + var wikiURL string + if repoInfo, ok := s.repoInfoCache.get(repoURL); ok { + if repoInfo.visibility == source_metadatapb.Visibility_private { + return true + } + wikiURL = wikiWebURLForRepoInfo(s.conn.GetEndpoint(), repoInfo) + } else { + var err error + wikiURL, err = wikiWebURLForRepoCloneURL(repoURL) + if err != nil { + return false + } + } req, err := http.NewRequestWithContext(ctx, http.MethodHead, wikiURL, nil) if err != nil { return false } - res, err := s.connector.APIClient().Client().Do(req) + res, err := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds).Do(req) if err != nil { return false } @@ -359,22 +402,30 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool { func (s *Source) normalizeRepo(repo string) (string, error) { // If it's a full URL (has protocol), normalize it - if regexp.MustCompile(`^[a-z]+://`).MatchString(repo) { - + if hasURLScheme(repo) { return giturl.NormalizeGithubRepo(repo) } + if isSCPStyleRepoURL(repo) { + repoURL, err := sourcegit.GitURLParse(repo) + if err == nil && repoURL.Host != "" && repoURL.Path != "" { + u := &url.URL{ + Scheme: "https", + Host: repoURL.Host, + Path: repoURL.Path, + } + return giturl.NormalizeGithubRepo(u.String()) + } + } // If it's a repository name (contains / but not http), convert to full URL first - if strings.Contains(repo, "/") && !regexp.MustCompile(`^[a-z]+://`).MatchString(repo) { + if strings.Contains(repo, "/") && !hasURLScheme(repo) { fullURL := "https://github.com/" + repo // If using GitHub Enterprise, adjust the URL accordingly - if s.conn != nil && s.conn.Endpoint != "" && !endsWithGithub.MatchString(s.conn.Endpoint) { - u, err := url.Parse(s.conn.Endpoint) + if s.conn != nil && s.conn.Endpoint != "" && !isGitHubCloudEndpoint(s.conn.Endpoint) { + u, err := endpointBaseURL(s.conn.Endpoint) if err != nil { return "", fmt.Errorf("invalid enterprise endpoint: %w", err) } - // we want to remove any path components from the endpoint and just use the host - u.Path = "/" + repo - fullURL = u.String() + fullURL = u.JoinPath(strings.Split(repo, "/")...).String() } return giturl.NormalizeGithubRepo(fullURL) } diff --git a/pkg/sources/github/target_gone_test.go b/pkg/sources/github/target_gone_test.go new file mode 100644 index 000000000000..082c680a5315 --- /dev/null +++ b/pkg/sources/github/target_gone_test.go @@ -0,0 +1,150 @@ +package github + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-github/v67/github" + "github.com/stretchr/testify/assert" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" +) + +func testClientForServer(t *testing.T, serverURL string) *github.Client { + t.Helper() + client := github.NewClient(nil) + base, err := url.Parse(serverURL + "/") + assert.NoError(t, err) + client.BaseURL = base + return client +} + +// downloadByPathServer answers the exact-path contents lookup, the raw +// download it points at, and the repository lookup repoReachable makes. +func downloadByPathServer(t *testing.T, fileStatus, rawStatus, repoStatus int) *httptest.Server { + t.Helper() + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/o/r/contents/dir/f.txt": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(fileStatus) + if fileStatus == http.StatusOK { + _, _ = fmt.Fprintf(w, `{"type":"file","name":"f.txt","path":"dir/f.txt","download_url":"%s/raw/dir/f.txt"}`, server.URL) + return + } + _, _ = w.Write([]byte(`{"message":"Not Found"}`)) + case "/raw/dir/f.txt": + w.WriteHeader(rawStatus) + if rawStatus == http.StatusOK { + _, _ = w.Write([]byte("file body")) + return + } + _, _ = w.Write([]byte("raw host error page")) + case "/repos/o/r": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(repoStatus) + if repoStatus == http.StatusOK { + _, _ = w.Write([]byte(`{"id":1,"name":"r","full_name":"o/r"}`)) + return + } + _, _ = w.Write([]byte(`{"message":"Not Found"}`)) + default: + t.Errorf("unexpected request: %s", r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + return server +} + +func TestDownloadContentsByPathFetchesFileBody(t *testing.T) { + server := downloadByPathServer(t, http.StatusOK, http.StatusOK, http.StatusOK) + defer server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + rc, resp, err := s.downloadContentsByPath(context.Background(), client, "o", "r", "dir/f.txt", "abc123") + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := io.ReadAll(rc) + assert.NoError(t, err) + assert.NoError(t, rc.Close()) + assert.Equal(t, "file body", string(body)) +} + +func TestDownloadContentsByPathReturns404Response(t *testing.T) { + server := downloadByPathServer(t, http.StatusNotFound, http.StatusOK, http.StatusOK) + defer server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + rc, resp, err := s.downloadContentsByPath(context.Background(), client, "o", "r", "dir/f.txt", "abc123") + assert.Error(t, err) + assert.Nil(t, rc) + // The 404 is the exact-path lookup's, which the caller combines with + // repoReachable to classify the target as gone. + assert.NotNil(t, resp) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) +} + +func TestDownloadContentsByPathNon404Failure(t *testing.T) { + server := downloadByPathServer(t, http.StatusForbidden, http.StatusOK, http.StatusOK) + defer server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + _, resp, err := s.downloadContentsByPath(context.Background(), client, "o", "r", "dir/f.txt", "abc123") + assert.Error(t, err) + assert.NotNil(t, resp) + assert.Equal(t, http.StatusForbidden, resp.StatusCode) +} + +func TestRepoReachable(t *testing.T) { + server := downloadByPathServer(t, http.StatusNotFound, http.StatusOK, http.StatusOK) + defer server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + assert.True(t, s.repoReachable(context.Background(), client, "o", "r")) +} + +func TestRepoNotReachable(t *testing.T) { + server := downloadByPathServer(t, http.StatusNotFound, http.StatusOK, http.StatusNotFound) + defer server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + // The repo 404s too: could be lost access rather than deletion, so the + // caller must not classify the target as gone. + assert.False(t, s.repoReachable(context.Background(), client, "o", "r")) +} + +func TestRepoReachableServerDown(t *testing.T) { + server := downloadByPathServer(t, http.StatusNotFound, http.StatusOK, http.StatusOK) + server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + assert.False(t, s.repoReachable(context.Background(), client, "o", "r")) +} + +func TestDownloadContentsByPathNon200Download(t *testing.T) { + server := downloadByPathServer(t, http.StatusOK, http.StatusInternalServerError, http.StatusOK) + defer server.Close() + + s := &Source{} + client := testClientForServer(t, server.URL) + rc, resp, err := s.downloadContentsByPath(context.Background(), client, "o", "r", "dir/f.txt", "abc123") + assert.Error(t, err) + assert.ErrorContains(t, err, "unexpected HTTP response status") + assert.Nil(t, rc) + // The returned response is the exact-path lookup's (200), not the failed + // download's, so the caller cannot mistake a download failure for the + // target being gone. + assert.NotNil(t, resp) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/pkg/sources/github/urls.go b/pkg/sources/github/urls.go new file mode 100644 index 000000000000..7b8ee7029d90 --- /dev/null +++ b/pkg/sources/github/urls.go @@ -0,0 +1,204 @@ +package github + +import ( + "fmt" + "net/url" + "strings" +) + +func wikiCloneURLForRepo(repoURL string) (string, bool) { + if !strings.HasSuffix(repoURL, ".git") || strings.HasSuffix(repoURL, ".wiki.git") { + return "", false + } + return strings.TrimSuffix(repoURL, ".git") + ".wiki.git", true +} + +func wikiCloneURLForRepoInfo(repoURL string, info repoInfo) (string, bool) { + return wikiCloneURLForRepoName(repoURL, info.name) +} + +func wikiCloneURLForRepoName(repoURL, name string) (string, bool) { + _, parts, err := getRepoURLParts(repoURL) + if err != nil || len(parts) != 3 || !strings.EqualFold(parts[2], name) || !strings.HasSuffix(repoURL, ".git") { + return "", false + } + return strings.TrimSuffix(repoURL, ".git") + ".wiki.git", true +} + +func repoCloneURLForWikiCloneURL(repoURL string) (string, bool) { + if strings.HasSuffix(repoURL, ".wiki.git") { + return strings.TrimSuffix(repoURL, ".wiki.git") + ".git", true + } + return "", false +} + +func repoURLsForInstallationLookup(repoURL string) []string { + repoURLs := []string{repoURL} + // Installation APIs return repository clone URLs, not wiki clone URLs. Keep + // the exact URL first so real repositories named "*.wiki" win before the + // wiki-parent fallback is attempted. + if parentRepoURL, ok := repoCloneURLForWikiCloneURL(repoURL); ok { + repoURLs = append(repoURLs, parentRepoURL) + } + return repoURLs +} + +func wikiWebURLForRepoInfo(endpoint string, info repoInfo) string { + return (&url.URL{ + Scheme: "https", + Host: githubWebHost(endpoint), + }).JoinPath(info.owner, info.name, "wiki").String() +} + +func wikiWebURLForRepoCloneURL(repoURL string) (string, error) { + u, err := url.Parse(repoURL) + if err != nil { + return "", fmt.Errorf("could not parse repo URL %q: %w", repoURL, err) + } + u.Path = strings.TrimSuffix(strings.TrimRight(u.Path, "/"), ".git") + return u.JoinPath("wiki").String(), nil +} + +func canonicalAPIEndpoint(endpoint string) string { + if isGitHubCloudEndpoint(endpoint) { + return cloudV3Endpoint + } + if !hasURLScheme(endpoint) { + return "https://" + strings.TrimLeft(endpoint, "/") + } + return endpoint +} + +func hasURLScheme(rawURL string) bool { + if !strings.Contains(rawURL, "://") { + return false + } + u, err := url.Parse(rawURL) + return err == nil && u.IsAbs() && u.Scheme != "" +} + +func isSCPStyleRepoURL(repoURL string) bool { + return !hasURLScheme(repoURL) && strings.Contains(repoURL, "@") && strings.Contains(repoURL, ":") +} + +func isGitHubCloudEndpoint(endpoint string) bool { + host, ok := endpointHost(endpoint) + if !ok { + return endpoint == "" + } + host = strings.TrimSuffix(strings.ToLower(host), ".") + if strings.HasPrefix(host, "[") { + // IPv6 endpoints are always Enterprise hosts. + return false + } + if hostWithoutPort, port, ok := strings.Cut(host, ":"); ok { + host = hostWithoutPort + if port != "" && port != "443" { + return false + } + } + return host == "github.com" || strings.HasSuffix(host, ".github.com") +} + +func endpointHost(endpoint string) (string, bool) { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "", false + } + + u, err := url.Parse(endpoint) + if err == nil && u.Host != "" { + return u.Host, true + } + + if strings.Contains(endpoint, "://") { + return "", false + } + host, _, _ := strings.Cut(strings.Trim(endpoint, "/"), "/") + return host, host != "" +} + +func endpointBaseURL(endpoint string) (*url.URL, error) { + host, ok := endpointHost(endpoint) + if !ok { + return nil, fmt.Errorf("invalid endpoint: %q", endpoint) + } + + scheme := "https" + if hasURLScheme(endpoint) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint: %q", endpoint) + } + scheme = u.Scheme + } + return &url.URL{Scheme: scheme, Host: host}, nil +} + +func repoCloneURLForTargetEndpoint(endpoint, repoURL string) (string, error) { + u, err := url.Parse(repoURL) + if err != nil { + return "", fmt.Errorf("could not parse target repository URL %q: %w", repoURL, err) + } + if u.Host == "" { + return "", fmt.Errorf("target repository URL %q has no host", repoURL) + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("target repository URL %q has unsupported scheme %q", repoURL, u.Scheme) + } + + endpointScheme := "https" + if hasURLScheme(endpoint) { + endpointURL, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("invalid endpoint: %q", endpoint) + } + if endpointURL.Scheme != "http" && endpointURL.Scheme != "https" { + return "", fmt.Errorf("endpoint %q has unsupported scheme %q", endpoint, endpointURL.Scheme) + } + endpointScheme = endpointURL.Scheme + } + + expectedHost := stripDefaultPort(githubWebHost(endpoint), endpointScheme) + actualHost := stripDefaultPort(u.Host, u.Scheme) + if !strings.EqualFold(actualHost, expectedHost) { + return "", fmt.Errorf("target repository host %q does not match GitHub endpoint host %q", u.Host, expectedHost) + } + + u.Scheme = endpointScheme + return u.String(), nil +} + +func stripDefaultPort(host, scheme string) string { + if strings.HasPrefix(host, "[") { + return host + } + hostWithoutPort, port, ok := strings.Cut(host, ":") + if !ok { + return host + } + if (scheme == "https" && port == "443") || (scheme == "http" && port == "80") { + return hostWithoutPort + } + return host +} + +func isWikiLink(link string) bool { + u, err := url.Parse(link) + if err != nil { + return false + } + segments := strings.Split(strings.Trim(u.Path, "/"), "/") + return len(segments) >= 3 && segments[2] == "wiki" +} + +func githubWebHost(endpoint string) string { + if isGitHubCloudEndpoint(endpoint) { + return "github.com" + } + host, ok := endpointHost(endpoint) + if !ok { + return "github.com" + } + return host +} diff --git a/pkg/sources/github/urls_test.go b/pkg/sources/github/urls_test.go new file mode 100644 index 000000000000..85bc50ee128c --- /dev/null +++ b/pkg/sources/github/urls_test.go @@ -0,0 +1,173 @@ +package github + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWikiCloneURLForRepo(t *testing.T) { + got, ok := wikiCloneURLForRepo("https://github.com/owner/repo.git") + require.True(t, ok) + assert.Equal(t, "https://github.com/owner/repo.wiki.git", got) + + _, ok = wikiCloneURLForRepo("https://github.com/owner/repo.wiki.git") + assert.False(t, ok) + + _, ok = wikiCloneURLForRepo("https://github.com/owner/repo") + assert.False(t, ok) +} + +func TestRepoCloneURLForWikiCloneURL(t *testing.T) { + got, ok := repoCloneURLForWikiCloneURL("https://github.com/owner/repo.wiki.git") + require.True(t, ok) + assert.Equal(t, "https://github.com/owner/repo.git", got) + + _, ok = repoCloneURLForWikiCloneURL("https://github.com/owner/repo.git") + assert.False(t, ok) +} + +func TestRepoURLsForInstallationLookup(t *testing.T) { + assert.Equal( + t, + []string{"https://github.com/owner/repo.git"}, + repoURLsForInstallationLookup("https://github.com/owner/repo.git"), + ) + assert.Equal( + t, + []string{"https://github.com/owner/repo.wiki.git", "https://github.com/owner/repo.git"}, + repoURLsForInstallationLookup("https://github.com/owner/repo.wiki.git"), + ) +} + +func TestWikiCloneURLForRepoInfo(t *testing.T) { + got, ok := wikiCloneURLForRepoInfo("https://github.com/owner/repo.wiki.git", repoInfo{name: "repo.wiki"}) + require.True(t, ok) + assert.Equal(t, "https://github.com/owner/repo.wiki.wiki.git", got) + + _, ok = wikiCloneURLForRepoInfo("https://github.com/owner/repo.wiki.git", repoInfo{name: "repo"}) + assert.False(t, ok) +} + +func TestWikiWebURLForRepoInfo(t *testing.T) { + info := repoInfo{owner: "owner", name: "repo"} + + assert.Equal(t, "https://github.com/owner/repo/wiki", wikiWebURLForRepoInfo("", info)) + assert.Equal( + t, + "https://github.company.local/owner/repo/wiki", + wikiWebURLForRepoInfo("https://github.company.local/api/v3", info), + ) +} + +func TestWikiWebURLForRepoCloneURL(t *testing.T) { + got, err := wikiWebURLForRepoCloneURL("https://github.com/owner/repo.git") + require.NoError(t, err) + assert.Equal(t, "https://github.com/owner/repo/wiki", got) + + got, err = wikiWebURLForRepoCloneURL("https://github.com/owner/repo") + require.NoError(t, err) + assert.Equal(t, "https://github.com/owner/repo/wiki", got) +} + +func TestHasURLScheme(t *testing.T) { + assert.True(t, hasURLScheme("https://user:pass@github.com/owner/repo.git")) + assert.True(t, hasURLScheme("ssh://git@github.com/owner/repo.git")) + assert.False(t, hasURLScheme("git@github.com:owner/repo.git")) + assert.False(t, hasURLScheme("owner/repo")) +} + +func TestIsSCPStyleRepoURL(t *testing.T) { + assert.True(t, isSCPStyleRepoURL("git@github.com:owner/repo.git")) + assert.False(t, isSCPStyleRepoURL("ssh://git@github.com/owner/repo.git")) + assert.False(t, isSCPStyleRepoURL("https://user:pass@github.com/owner/repo.git")) + assert.False(t, isSCPStyleRepoURL("owner/repo")) +} + +func TestEndpointHost(t *testing.T) { + tests := []struct { + name string + endpoint string + want string + wantOK bool + }{ + {name: "empty", endpoint: "", wantOK: false}, + {name: "cloud URL", endpoint: "https://api.github.com", want: "api.github.com", wantOK: true}, + {name: "cloud host", endpoint: "api.github.com", want: "api.github.com", wantOK: true}, + {name: "enterprise URL with path", endpoint: "https://github.example.com/api/v3", want: "github.example.com", wantOK: true}, + {name: "enterprise host with path", endpoint: "github.example.com/api/v3", want: "github.example.com", wantOK: true}, + {name: "host with port", endpoint: "https://github.example.com:8443/api/v3", want: "github.example.com:8443", wantOK: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := endpointHost(tt.endpoint) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestIsGitHubCloudEndpoint(t *testing.T) { + assert.True(t, isGitHubCloudEndpoint("")) + assert.True(t, isGitHubCloudEndpoint("github.com")) + assert.True(t, isGitHubCloudEndpoint("https://api.github.com")) + assert.True(t, isGitHubCloudEndpoint("api.github.com")) + assert.True(t, isGitHubCloudEndpoint("https://github.com:443")) + assert.False(t, isGitHubCloudEndpoint("https://legithub.com/api/v3")) + assert.False(t, isGitHubCloudEndpoint("https://github.example.com/api/v3")) + assert.False(t, isGitHubCloudEndpoint("https://enterprise.example.com/api/github.com")) + assert.False(t, isGitHubCloudEndpoint("https://github.com:8443/api/v3")) +} + +func TestEndpointBaseURL(t *testing.T) { + got, err := endpointBaseURL("https://github.example.com/api/v3") + require.NoError(t, err) + assert.Equal(t, "https://github.example.com", got.String()) + + got, err = endpointBaseURL("github.example.com/api/v3") + require.NoError(t, err) + assert.Equal(t, "https://github.example.com", got.String()) + + got, err = endpointBaseURL("ghe.example:8443/api/v3") + require.NoError(t, err) + assert.Equal(t, "https://ghe.example:8443", got.String()) +} + +func TestCanonicalAPIEndpoint(t *testing.T) { + assert.Equal(t, cloudV3Endpoint, canonicalAPIEndpoint("")) + assert.Equal(t, cloudV3Endpoint, canonicalAPIEndpoint("github.com")) + assert.Equal(t, cloudV3Endpoint, canonicalAPIEndpoint("https://api.github.com")) + assert.Equal(t, "https://github.example.com/api/v3", canonicalAPIEndpoint("github.example.com/api/v3")) + assert.Equal(t, "https://github.example.com/api/v3", canonicalAPIEndpoint("https://github.example.com/api/v3")) + assert.Equal(t, "https://github.com:8443/api/v3", canonicalAPIEndpoint("github.com:8443/api/v3")) +} + +func TestRepoCloneURLForTargetEndpoint(t *testing.T) { + got, err := repoCloneURLForTargetEndpoint("", "http://github.com/owner/repo.git") + require.NoError(t, err) + assert.Equal(t, "https://github.com/owner/repo.git", got) + + got, err = repoCloneURLForTargetEndpoint(cloudV3Endpoint, "https://github.com:443/owner/repo.git") + require.NoError(t, err) + assert.Equal(t, "https://github.com:443/owner/repo.git", got) + + got, err = repoCloneURLForTargetEndpoint("http://github.company/api/v3", "https://github.company/owner/repo.git") + require.NoError(t, err) + assert.Equal(t, "http://github.company/owner/repo.git", got) + + _, err = repoCloneURLForTargetEndpoint(cloudV3Endpoint, "https://attacker.example/owner/repo.git") + require.Error(t, err) + assert.Contains(t, err.Error(), "does not match GitHub endpoint host") + + _, err = repoCloneURLForTargetEndpoint(cloudV3Endpoint, "ssh://git@github.com/owner/repo.git") + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported scheme") +} + +func TestIsWikiLink(t *testing.T) { + assert.True(t, isWikiLink("https://github.com/owner/repo/wiki/path/to/file")) + assert.False(t, isWikiLink("https://github.com/owner/repo/blob/main/path/to/file")) + assert.False(t, isWikiLink("not a url")) +} diff --git a/pkg/sources/github_experimental/object_discovery.go b/pkg/sources/github_experimental/object_discovery.go index a09d8415e00e..991ec46a6680 100644 --- a/pkg/sources/github_experimental/object_discovery.go +++ b/pkg/sources/github_experimental/object_discovery.go @@ -209,7 +209,7 @@ func writeCommitsToDisk(commits []string, commitsType, folder string) error { if err != nil { return err } - defer file.Close() + defer func() { _ = file.Close() }() for _, commit := range commits { if _, err := file.WriteString(commit + "\n"); err != nil { @@ -413,7 +413,7 @@ func checkHashes(ctx context.Context, client *github.Client, owner, repo string, if err != nil { return nil, fmt.Errorf("python request error: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { @@ -564,7 +564,7 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan return fmt.Errorf("failed to clone the repository: %w", err) } - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() // count total valid hashes validHashes, err := getExistingHashes(path) diff --git a/pkg/sources/github_experimental/repo.go b/pkg/sources/github_experimental/repo.go index 3b4bf67601cd..46bf7d2d6353 100644 --- a/pkg/sources/github_experimental/repo.go +++ b/pkg/sources/github_experimental/repo.go @@ -2,7 +2,7 @@ package github_experimental import ( "fmt" - "regexp" + regexp "github.com/wasilibs/go-re2" "strings" "sync" diff --git a/pkg/sources/gitlab/README.md b/pkg/sources/gitlab/README.md new file mode 100644 index 000000000000..90c0ad8bd6f9 --- /dev/null +++ b/pkg/sources/gitlab/README.md @@ -0,0 +1,239 @@ +# GitLab Source + +## Overview + +The GitLab source lets TruffleHog scan GitLab projects for secrets, credentials, and sensitive data. It uses the GitLab API to find projects, then clones and scans each one with the git source underneath. + +## GitLab Fundamentals + +### What does this source scan? + +This source finds projects through the GitLab API (by an explicit list, by group, or by listing everything the account can see), then clones and scans each project the same way the git source does. + +### Key GitLab Terminology + +| Term | Description | +|------|-------------| +| **Project** | A single GitLab repository, called a project in GitLab's own terms | +| **Group** | A GitLab container that holds projects and can also hold subgroups | +| **Subgroup** | A group nested inside another group | +| **Personal Access Token** | A credential tied to a GitLab account, used instead of a username and password | +| **GitLab Cloud** | GitLab's own hosted service at gitlab.com | +| **Self Managed GitLab** | A GitLab instance hosted by the customer on their own infrastructure | + +## Features + +- **Multiple Authentication Methods**: Personal access token, OAuth refresh token, or username and password +- **Multiple Scan Targets**: Scan by an explicit list of projects, by group, or every project the account can see +- **Project Filtering**: Include or exclude projects using glob patterns +- **Self Managed Support**: Works with self managed GitLab instances in addition to gitlab.com +- **Concurrent Processing**: Projects are cloned and scanned in parallel + +## Configuration + +### Authentication Methods + +#### 1. Personal Access Token + +**CLI Usage:** +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitLab + token: "glpat-xxxxxxxxxxxxxxxxxxxx" + name: gitlab-scan + type: SOURCE_TYPE_GITLAB + verify: true +``` + +On the CLI, the token can also be set with the `GITLAB_TOKEN` environment variable instead of `--token`. This does not apply to a YAML config file, since TruffleHog does not read environment variables while loading YAML. A token is always required, there is no unauthenticated scanning mode for GitLab. + +--- + +#### 2. OAuth Refresh Token + +This method is only available in the YAML config, not the CLI. Only the refresh token is used to authenticate, the other OAuth fields are not read by this source. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitLab + oauth: + refresh_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + name: gitlab-scan + type: SOURCE_TYPE_GITLAB + verify: true +``` + +--- + +#### 3. Username and Password + +This method is only available in the YAML config, not the CLI. If the password is actually a personal access token rather than a real password, TruffleHog notices basic authentication fails and falls back to using it as a token instead. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.GitLab + basic_auth: + username: my-username + password: my-password + name: gitlab-scan + type: SOURCE_TYPE_GITLAB + verify: true +``` + +### Choosing What to Scan + +**Scan Specific Projects** + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --repo https://gitlab.com/my-org/my-project +``` + +`--repo` can be repeated. When projects are given directly, group and project filters are not used, and the include and exclude patterns are ignored for them. + +**Scan a Group** + +Scans every project in the group and its subgroups. + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --group-id 12345678 +``` + +`--group-id` can be repeated to scan more than one group. It cannot be combined with `--repo`. + +**Scan Everything the Account Can See** + +If neither `--repo` nor `--group-id` is given, TruffleHog lists every project the token, OAuth account, or username and password can access. + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx +``` + +On gitlab.com, this only lists projects the account is a member of. On a self managed instance, it lists every project the account can see, not just ones it is a member of. + +### Filtering What Gets Scanned + +**Include or Exclude Projects** + +Only applies when scanning a group or everything the account can see, not to an explicit `--repo` list. A project can be filtered out by either pattern; there is no include-wins rule here, being excluded always wins. + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --include-repos "my-org/prod-*" +``` + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --exclude-repos "my-org/test-*" +``` + +**Include or Exclude Files** + +Both flags take a path to a file that holds one regex per line. Blank lines and lines starting with `#` are ignored. + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --include-paths include-patterns.txt +``` + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --exclude-paths exclude-patterns.txt +``` + +**Number of Workers** + +There is no way to configure this for GitLab. TruffleHog's global `--concurrency` flag is not used by this source, projects are always cloned and scanned in parallel across as many workers as the machine has CPU cores. + +### Clone Location and Cleanup + +By default, each project is cloned into a temporary directory and deleted after scanning. `--clone-path` clones into a given directory instead, and `--no-cleanup` (only valid together with `--clone-path`) keeps the clones on disk afterward instead of deleting them. + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --clone-path /path/to/clones --no-cleanup +``` + +### Self Managed GitLab + +Point at a self managed GitLab instance instead of gitlab.com. Only `https` is accepted. + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --endpoint https://gitlab.mycompany.com +``` + +## How Scanning Works + +### Scanning Process + +1. **Enumeration**: Depending on the scan target, lists the explicit projects given, every project in the given groups and their subgroups, or every project the account can see, applying the include and exclude filters when listing by group or by account +2. **Project Cloning**: Clones each enumerated project using the git source, the same way the git source scans a remote repository +3. **Commit Scanning**: Scans every commit in the clone for secrets, the same way the git source does +4. **Progress Tracking**: Keeps track of which projects have already been scanned, so a scan can be resumed without redoing work + +### What Gets Scanned + +- Every commit in every project matching the scan target and filters +- Projects shared into a scanned group from elsewhere, when scanning with `--group-id`. This is always on for a group scan; the YAML `exclude_projects_shared_into_groups` field exists but has no effect under TruffleHog's default GitLab enumeration mode + +### What Doesn't Get Scanned + +- Projects excluded by `--include-repos` or `--exclude-repos`, when scanning a group or the whole account +- Projects TruffleHog does not have access to with the configured credentials + +## Usage Examples + +### Scanning Everything the Account Can See + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx +``` + +### Scanning Specific Projects + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --repo https://gitlab.com/my-org/project-one --repo https://gitlab.com/my-org/project-two +``` + +### Scanning a Group + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --group-id 12345678 +``` + +### Scanning Self Managed GitLab + +```bash +trufflehog gitlab --token glpat-xxxxxxxxxxxxxxxxxxxx --endpoint https://gitlab.mycompany.com +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Authentication failures when connecting to GitLab +**Solution**: Check that the token, OAuth refresh token, or username and password are correct and have permission to read the target projects. + +--- + +**Issue**: `invalid config: you cannot specify both repositories and groups at the same time` +**Solution**: Use either `--repo` or `--group-id`, not both together. + +--- + +**Issue**: `https was not used as URL scheme, but is required. Please use https` +**Solution**: A self managed `--endpoint` must use `https`, `http` is rejected. + +--- + +**Issue**: `received error on listing projects, you might not have permissions to do that` +**Solution**: The account used to authenticate does not have permission to list projects this way. Use `--repo` with explicit project URLs instead. + +--- + +**Issue**: No projects found when scanning by group or by account +**Solution**: Check the include and exclude patterns are not excluding every project, and that the token's account actually has access to the expected projects. diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index c13cda6f8d77..d4a531376efc 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -1,6 +1,7 @@ package gitlab import ( + "errors" "fmt" "net/url" "os" @@ -10,6 +11,15 @@ import ( "sync" "time" + gogit "github.com/go-git/go-git/v5" + "github.com/gobwas/glob" + "github.com/hashicorp/golang-lru/v2/expirable" + gitlab "gitlab.com/gitlab-org/api/client-go" + "golang.org/x/oauth2" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/feature" @@ -20,20 +30,12 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" - - gogit "github.com/go-git/go-git/v5" - "github.com/gobwas/glob" - gitlab "gitlab.com/gitlab-org/api/client-go" - "golang.org/x/oauth2" - "golang.org/x/sync/errgroup" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" ) const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB -// This is the URL for gitlab hosted at gitlab.com -const gitlabBaseURL = "https://gitlab.com/" +// Base URL for GitLab Cloud (hosted at gitlab.com) +const gitlabCloudBaseURL = "https://gitlab.com/" type Source struct { name string @@ -72,16 +74,22 @@ type Source struct { sources.CommonSourceUnitUnmarshaller useAuthInUrl bool - - clonePath string - noCleanup bool + clonePath string + noCleanup bool printLegacyJSON bool projectsPerPage int64 // cache of repo URL to project info, used when generating metadata for chunks - repoToProjCache repoToProjectCache + projectMetadataCache *expirable.LRU[string, *projectMetadata] +} + +// projectMetadata represents GitLab project metadata. +type projectMetadata struct { + id int64 + name string + owner string } // WithCustomContentWriter sets the useCustomContentWriter flag on the source. @@ -242,11 +250,23 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou Timestamp: sanitizer.UTF8(info.Timestamp), Line: info.Line, } - proj, ok := s.repoToProjCache.get(info.Repository) + // check for project metadata in the cache + project, ok := s.projectMetadataCache.Get(info.Repository) + if !ok { + // Long-running scans of a single large repo can outlive the + // cache TTL between the initial ensureProjectInCache call and + // later chunks from that same repo. Repopulate on miss. + s.ensureProjectInCache(ctx, info.Repository) + project, ok = s.projectMetadataCache.Get(info.Repository) + } if ok { - gitlabMetadata.ProjectId = int64(proj.id) - gitlabMetadata.ProjectName = proj.name - gitlabMetadata.ProjectOwner = proj.owner + ctx.Logger().V(5).Info("cache hit inside source metadata func: found project metadata in the cache", "cache_key", info.Repository) + gitlabMetadata.ProjectId = project.id + gitlabMetadata.ProjectName = project.name + gitlabMetadata.ProjectOwner = project.owner + } else { + ctx.Logger().Error(errors.New("failed to get repo metadata from cache"), + "cache miss: not found project metadata in the cache", "cache_key", info.Repository) } return &source_metadatapb.MetaData{ @@ -260,9 +280,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou } s.git = git.NewGit(cfg) - s.repoToProjCache = repoToProjectCache{ - cache: make(map[string]*project), - } + s.projectMetadataCache = expirable.NewLRU[string, *projectMetadata]( + 15000, // upto 15000 entries + nil, + 60*time.Minute, // time-based expiration - 1 hour + ) return nil } @@ -314,8 +336,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar } else { gitlabReposEnumerated.WithLabelValues(s.name).Set(float64(len(repos))) - // ensure project details for specified repos are cached - // this is required to populate metadata during chunking + // Ensure project details for the specified repositories are cached. + // This is required to populate metadata during chunking. + // Note: Repository URLs are already normalized, so the cache check + // uses the normalized repo URL directly. for _, repo := range repos { s.ensureProjectInCache(ctx, repo) } @@ -586,7 +610,10 @@ func (s *Source) getAllProjectRepos( } // Report the unit. ctx.Logger().V(3).Info("accepting project") - s.cacheGitlabProject(proj) + + // Cache the GitLab project metadata. + s.cacheGitlabProjectMetadata(ctx, proj) + unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo} gitlabReposEnumerated.WithLabelValues(s.name).Inc() projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace) @@ -630,7 +657,7 @@ func (s *Source) getAllProjectRepos( Owned: gitlab.Ptr(false), } - if s.url != gitlabBaseURL { + if s.url != gitlabCloudBaseURL { listGroupsOptions.AllAvailable = gitlab.Ptr(true) } @@ -722,7 +749,7 @@ func (s *Source) getAllProjectReposV2( } // for gitlab.com instance, include only projects where the user is a member. - if s.url == gitlabBaseURL { + if s.url == gitlabCloudBaseURL { projectQueryOptions.Membership = gitlab.Ptr(true) } @@ -783,7 +810,9 @@ func (s *Source) getAllProjectReposV2( // report the unit. projCtx.Logger().V(3).Info("accepting project") - s.cacheGitlabProject(project) + // Cache the GitLab project metadata. + s.cacheGitlabProjectMetadata(projCtx, project) + unit := git.SourceUnit{Kind: git.UnitRepo, ID: project.HTTPURLToRepo} gitlabReposEnumerated.WithLabelValues(s.name).Inc() @@ -837,7 +866,7 @@ func (s *Source) getAllProjectReposInGroups( } // For non gitlab.com instances, you might want to adjust access levels - if s.url != gitlabBaseURL { + if s.url != gitlabCloudBaseURL { projectOpts.MinAccessLevel = gitlab.Ptr(gitlab.GuestPermissions) } @@ -899,7 +928,9 @@ func (s *Source) getAllProjectReposInGroups( // report the unit. projCtx.Logger().V(3).Info("accepting project") - s.cacheGitlabProject(proj) + // Cache the GitLab project metadata. + s.cacheGitlabProjectMetadata(projCtx, proj) + unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo} gitlabReposEnumerated.WithLabelValues(s.name).Inc() projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace) @@ -977,10 +1008,15 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) // if legacy JSON is enabled, don't remove the directory because we need it for outputting legacy JSON. if !s.printLegacyJSON { if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.noCleanup && s.clonePath != "") { - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() } } + // Re-populate the cache if the entry expired since it was first + // cached during enumeration; scanRepos can run for hours on + // large orgs, longer than the cache's TTL. + s.ensureProjectInCache(ctx, repoURL) + logger.V(2).Info("starting scan", "num", i+1, "total", len(s.repos)) if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, sources.ChanReporter{Ch: chunksChan}); err != nil { scanErrs.Add(err) @@ -1022,77 +1058,6 @@ func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) { s.scanOptions = scanOptions } -func buildIgnorer(include, exclude []string, onCompile func(err error, pattern string)) func(repo string) bool { - - // compile and load globRepoFilter - globRepoFilter := newGlobRepoFilter(include, exclude, onCompile) - - f := func(repo string) bool { - if !globRepoFilter.includeRepo(repo) || globRepoFilter.ignoreRepo(repo) { - return true - } - return false - } - - return f -} - -func normalizeRepos(repos []string) ([]string, []error) { - // Optimistically allocate space for all valid repositories. - validRepos := make([]string, 0, len(repos)) - var errs []error - for _, prj := range repos { - repo, err := giturl.NormalizeGitlabRepo(prj) - if err != nil { - errs = append(errs, fmt.Errorf("unable to normalize gitlab repo url %q: %w", prj, err)) - continue - } - - validRepos = append(validRepos, repo) - } - return validRepos, errs -} - -// normalizeGitlabEndpoint ensures that if an endpoint is going to gitlab.com, we use https://gitlab.com/ as the endpoint. -// If we see the protocol is http, we error, because this shouldn't be used. -// Otherwise, it ensures we are using https as our protocol, if none was provided. -func normalizeGitlabEndpoint(gitlabEndpoint string) (string, error) { - if gitlabEndpoint == "" { - return gitlabBaseURL, nil - } - - gitlabURL, err := url.Parse(gitlabEndpoint) - if err != nil { - return "", err - } - - // We probably didn't receive a URL with a scheme, which messed up the parsing. - if gitlabURL.Host == "" { - gitlabURL, err = url.Parse("https://" + gitlabEndpoint) - if err != nil { - return "", err - } - } - - // If the host is gitlab.com, this is the cloud version, which has only one valid endpoint. - if gitlabURL.Host == "gitlab.com" { - return gitlabBaseURL, nil - } - - // Beyond here, on-prem gitlab is being used, so we have to mostly leave things as-is. - - if gitlabURL.Scheme != "https" { - return "", fmt.Errorf("https was not used as URL scheme, but is required. Please use https") - } - - // The gitlab library wants trailing slashes. - if !strings.HasSuffix(gitlabURL.Path, "/") { - gitlabURL.Path = gitlabURL.Path + "/" - } - - return gitlabURL.String(), nil -} - // Enumerate reports all GitLab repositories to be scanned to the reporter. If // none are configured, it will find all repositories within all projects that // the configured user has access to, while respecting the configured ignore @@ -1157,11 +1122,19 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte return nil } + // Normalize before cloning so the git remote URL (used as the cache key + // when attaching metadata to chunks) matches the key used when caching + // project details below. + normalizedRepoURL, err := giturl.NormalizeGitlabRepo(repoURL) + if err != nil { + ctx.Logger().Error(err, "failed to normalize GitLab Repo", "repo", repoURL) + return err + } + var path string var repo *gogit.Repository - var err error if s.authMethod == "UNAUTHENTICATED" { - path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL, s.clonePath) + path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, normalizedRepoURL, s.clonePath) } else { // If a username is not provided we need to use a default one in order to clone a private repo. // Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user. @@ -1170,7 +1143,7 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte user = "placeholder" } - path, repo, err = git.CloneRepoUsingToken(ctx, s.token, repoURL, s.clonePath, user, s.useAuthInUrl) + path, repo, err = git.CloneRepoUsingToken(ctx, s.token, normalizedRepoURL, s.clonePath, user, s.useAuthInUrl) } if err != nil { return err @@ -1180,34 +1153,36 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte // if legacy JSON is enabled, don't remove the directory because we need it for outputting legacy JSON. if !s.printLegacyJSON { if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.noCleanup && s.clonePath != "") { - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() } } // ensure project details are cached // this is required to populate metadata during chunking - s.ensureProjectInCache(ctx, repoURL) + s.ensureProjectInCache(ctx, normalizedRepoURL) return s.git.ScanRepo(ctx, repo, path, s.scanOptions, reporter) } -// ensureProjectInCache checks if the project for the given repo URL is in the cache, -// and if not, queries the GitLab API to fetch the project and adds it to the cache. -func (s *Source) ensureProjectInCache(ctx context.Context, repoUrl string) { - // check if project is already in cache - if _, ok := s.repoToProjCache.get(repoUrl); ok { +// ensureProjectInCache ensures that the project for the given repository URL +// exists in the cache. If not, it fetches the project from the GitLab API +// and stores it in the cache. +func (s *Source) ensureProjectInCache(ctx context.Context, repoURL string) { + // Check if the project is already cached. + if _, ok := s.projectMetadataCache.Get(repoURL); ok { + ctx.Logger().V(5).Info("cache hit: found project metadata in the cache", "cache_key", repoURL) return } - // query project - proj, err := s.getGitlabProject(ctx, repoUrl) + // Fetch the project from GitLab. + project, err := s.getGitlabProject(ctx, repoURL) if err != nil { - ctx.Logger().Error(err, "could not fetch project for repo", "repo", repoUrl) + ctx.Logger().Error(err, "failed to fetch GitLab project", "repo", repoURL) return } - // add to cache - s.cacheGitlabProject(proj) + // Cache the project metadata. + s.cacheGitlabProjectMetadata(ctx, project) } func (s *Source) getGitlabProject(ctx context.Context, repoUrl string) (*gitlab.Project, error) { @@ -1230,16 +1205,100 @@ func (s *Source) getGitlabProject(ctx context.Context, repoUrl string) (*gitlab. return proj, nil } -func (s *Source) cacheGitlabProject(gitlabProj *gitlab.Project) { - proj := &project{ - id: gitlabProj.ID, - name: gitlabProj.NameWithNamespace, +// cacheGitlabProjectMetadata caches GitLab project metadata keyed by the +// normalized GitLab repository URL. +func (s *Source) cacheGitlabProjectMetadata(ctx context.Context, glProject *gitlab.Project) { + proj := &projectMetadata{ + id: int64(glProject.ID), + name: glProject.NameWithNamespace, + } + + if glProject.Owner != nil { + if email := glProject.Owner.Email; email != "" { + proj.owner = email + } else { + proj.owner = glProject.Owner.Username + } + } + + repoURL, err := giturl.NormalizeGitlabRepo(glProject.HTTPURLToRepo) + if err != nil { + ctx.Logger().Error(err, "failed to normalize GitLab Repo", "repo", glProject.HTTPURLToRepo) + return + } + + ctx.Logger().V(5).Info("cache set: added project metadata in the cache", "cache_key", repoURL) + s.projectMetadataCache.Add(repoURL, proj) +} + +func buildIgnorer(include, exclude []string, onCompile func(err error, pattern string)) func(repo string) bool { + + // compile and load globRepoFilter + globRepoFilter := newGlobRepoFilter(include, exclude, onCompile) + + f := func(repo string) bool { + if !globRepoFilter.includeRepo(repo) || globRepoFilter.ignoreRepo(repo) { + return true + } + return false + } + + return f +} + +// normalizeRepos convert the repo urls from https://gitlab.com/org/repo -> https://gitlab.com/org/repo.git +func normalizeRepos(repos []string) ([]string, []error) { + // Optimistically allocate space for all valid repositories. + validRepos := make([]string, 0, len(repos)) + var errs []error + for _, prj := range repos { + repo, err := giturl.NormalizeGitlabRepo(prj) + if err != nil { + errs = append(errs, fmt.Errorf("unable to normalize gitlab repo url %q: %w", prj, err)) + continue + } + + validRepos = append(validRepos, repo) + } + return validRepos, errs +} + +// normalizeGitlabEndpoint ensures that if an endpoint is going to gitlab.com, we use https://gitlab.com/ as the endpoint. +// If we see the protocol is http, we error, because this shouldn't be used. +// Otherwise, it ensures we are using https as our protocol, if none was provided. +func normalizeGitlabEndpoint(gitlabEndpoint string) (string, error) { + if gitlabEndpoint == "" { + return gitlabCloudBaseURL, nil } - if gitlabProj.Owner != nil { - proj.owner = gitlabProj.Owner.Email - if proj.owner == "" { - proj.owner = gitlabProj.Owner.Username + + gitlabURL, err := url.Parse(gitlabEndpoint) + if err != nil { + return "", err + } + + // We probably didn't receive a URL with a scheme, which messed up the parsing. + if gitlabURL.Host == "" { + gitlabURL, err = url.Parse("https://" + gitlabEndpoint) + if err != nil { + return "", err } } - s.repoToProjCache.set(gitlabProj.HTTPURLToRepo, proj) + + // If the host is gitlab.com, this is the cloud version, which has only one valid endpoint. + if gitlabURL.Host == "gitlab.com" { + return gitlabCloudBaseURL, nil + } + + // Beyond here, on-prem gitlab is being used, so we have to mostly leave things as-is. + + if gitlabURL.Scheme != "https" { + return "", fmt.Errorf("https was not used as URL scheme, but is required. Please use https") + } + + // The gitlab library wants trailing slashes. + if !strings.HasSuffix(gitlabURL.Path, "/") { + gitlabURL.Path = gitlabURL.Path + "/" + } + + return gitlabURL.String(), nil } diff --git a/pkg/sources/gitlab/gitlab_integration_test.go b/pkg/sources/gitlab/gitlab_integration_test.go index 423acac29ae6..c07c43172e53 100644 --- a/pkg/sources/gitlab/gitlab_integration_test.go +++ b/pkg/sources/gitlab/gitlab_integration_test.go @@ -794,7 +794,7 @@ func TestSource_Enumerate_ProjectDetailsInChunkMetadata(t *testing.T) { chunksCh := make(chan *sources.Chunk, 1) chanReporter := sources.ChanReporter{Ch: chunksCh} // Clear cache to force querying project details - clear(s.repoToProjCache.cache) + s.projectMetadataCache.Purge() go func() { defer close(chunksCh) for _, unit := range testReporter.Units { @@ -859,7 +859,7 @@ func TestSource_Chunks_SimplifiedGitlabEnumeration(t *testing.T) { require.NotEmpty(t, testReporter.Units, "enumeration returned no units") // Clear project cache to force project-detail lookups during chunking - clear(s.repoToProjCache.cache) + s.projectMetadataCache.Purge() // Channel-based reporter to capture emitted chunks chunksCh := make(chan *sources.Chunk, 1) diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go index 5a0848fc4904..8eb4642a30a8 100644 --- a/pkg/sources/gitlab/gitlab_test.go +++ b/pkg/sources/gitlab/gitlab_test.go @@ -141,24 +141,24 @@ func Test_normalizeGitlabEndpoint(t *testing.T) { wantErr bool }{ "the cloud url should return the cloud url": { - inputEndpoint: gitlabBaseURL, - outputEndpoint: gitlabBaseURL, + inputEndpoint: gitlabCloudBaseURL, + outputEndpoint: gitlabCloudBaseURL, }, "empty string should return the cloud url": { inputEndpoint: "", - outputEndpoint: gitlabBaseURL, + outputEndpoint: gitlabCloudBaseURL, }, "no scheme cloud url should return the cloud url": { inputEndpoint: "gitlab.com", - outputEndpoint: gitlabBaseURL, + outputEndpoint: gitlabCloudBaseURL, }, "no scheme cloud url with trailing slash should return the cloud url": { inputEndpoint: "gitlab.com/", - outputEndpoint: gitlabBaseURL, + outputEndpoint: gitlabCloudBaseURL, }, "http scheme cloud url with organization should return the cloud url": { inputEndpoint: "http://gitlab.com/trufflesec", - outputEndpoint: gitlabBaseURL, + outputEndpoint: gitlabCloudBaseURL, }, // On-prem endpoint testing. "on-prem url should be unchanged": { diff --git a/pkg/sources/gitlab/project_cache.go b/pkg/sources/gitlab/project_cache.go deleted file mode 100644 index 0fe492c1365b..000000000000 --- a/pkg/sources/gitlab/project_cache.go +++ /dev/null @@ -1,29 +0,0 @@ -package gitlab - -import "sync" - -type project struct { - id int64 - name string - owner string -} - -type repoToProjectCache struct { - sync.RWMutex - - cache map[string]*project -} - -func (r *repoToProjectCache) get(repo string) (*project, bool) { - r.RLock() - defer r.RUnlock() - proj, ok := r.cache[repo] - return proj, ok -} - -func (r *repoToProjectCache) set(repo string, proj *project) { - r.Lock() - defer r.Unlock() - - r.cache[repo] = proj -} diff --git a/pkg/sources/huggingface/README.md b/pkg/sources/huggingface/README.md new file mode 100644 index 000000000000..764415073bd6 --- /dev/null +++ b/pkg/sources/huggingface/README.md @@ -0,0 +1,195 @@ +# HuggingFace Source + +## Overview + +The HuggingFace source lets TruffleHog scan HuggingFace models, spaces, datasets, and buckets for secrets, credentials, and sensitive data. It uses the HuggingFace API to find repositories, then clones and scans each one with the git source underneath. Buckets are handled differently since they are plain file storage, not git repositories. + +## HuggingFace Fundamentals + +### What does this source scan? + +This source finds models, spaces, and datasets through the HuggingFace API (by an explicit list, or by organization and user), then clones and scans each one the same way the git source does. Buckets are found the same way but are downloaded and scanned file by file instead of being cloned, since they are not git repositories. + +### Key HuggingFace Terminology + +| Term | Description | +|------|-------------| +| **Model** | A trained machine learning model hosted on HuggingFace, stored as a git repository | +| **Space** | An application or demo hosted on HuggingFace, stored as a git repository | +| **Dataset** | A collection of data hosted on HuggingFace, stored as a git repository | +| **Bucket** | Plain object storage hosted on HuggingFace, not a git repository | +| **Organization** | A HuggingFace account that holds models, spaces, datasets, and buckets owned by a team | +| **Discussion** | A conversation thread attached to a model, space, or dataset | + +## Features + +- **Multiple Scan Targets**: Scan models, spaces, datasets, and buckets, by an explicit list or by organization and user +- **Multiple Authentication Methods**: Personal access token, or unauthenticated for public repositories +- **Repository Filtering**: Include or exclude models, spaces, datasets, and buckets using glob patterns, or skip an entire resource type +- **Discussion and Pull Request Scanning**: Optionally scan the comments on discussions and pull requests +- **Concurrent Processing**: Repositories and bucket files are cloned, downloaded, and scanned in parallel across a configurable number of workers + +## Configuration + +Note: as of this writing, `pkg/config/config.go` does not have a case for `SOURCE_TYPE_HUGGINGFACE` in its source loader, so a YAML `--config` file cannot be used for HuggingFace. Use the CLI flags or environment variable below instead. + +### Authentication Methods + +#### 1. Personal Access Token + +**CLI Usage:** +```bash +trufflehog huggingface --org my-org --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +The token can also be set with the `HUGGINGFACE_TOKEN` environment variable instead of `--token`. + +--- + +#### 2. Unauthenticated + +Only works for public models, spaces, datasets, and buckets. + +**CLI Usage:** +```bash +trufflehog huggingface --org my-org +``` + +### Choosing What to Scan + +At least one of `--org`, `--user`, `--model`, `--space`, `--dataset`, or `--bucket` must be given. + +**Scan an Organization or User** + +```bash +trufflehog huggingface --org my-org --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +```bash +trufflehog huggingface --user my-user --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +This finds every model, space, dataset, and bucket owned by the organization or user. Use `--skip-all-models`, `--skip-all-spaces`, `--skip-all-datasets`, or `--skip-all-buckets` to leave out a whole resource type. + +```bash +trufflehog huggingface --org my-org --skip-all-buckets --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +**Scan Specific Models, Spaces, Datasets, or Buckets** + +```bash +trufflehog huggingface --model my-org/my-model --space my-org/my-space --dataset my-org/my-dataset --bucket my-org/my-bucket --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +Each of these flags can be repeated, and each name should be in `owner/name` form so TruffleHog can build the right URL for it. Unlike `--include-*` and `--ignore-*`, TruffleHog does not check this format for `--model`, `--space`, `--dataset`, or `--bucket` themselves, a name without a slash just produces a broken URL later instead of a clear error. When a model, space, dataset, or bucket is given directly this way, `--include-*` and `--ignore-*` filters are not applied to it. + +### Filtering What Gets Scanned + +**Include or Exclude Models, Spaces, Datasets, or Buckets** + +These only apply when scanning with `--org` or `--user`, not to resources given directly with `--model`, `--space`, `--dataset`, or `--bucket`. Each name must be in `owner/name` form, and can also be a glob pattern. + +```bash +trufflehog huggingface --org my-org --include-models "my-org/prod-*" --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +```bash +trufflehog huggingface --org my-org --ignore-datasets "my-org/test-*" --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +The same pattern works for `--include-spaces`, `--ignore-spaces`, `--include-buckets`, and `--ignore-buckets`. + +**Number of Workers** + +There is no HuggingFace specific flag for this. Use TruffleHog's global `--concurrency` flag to set how many repositories and bucket files are processed at the same time. + +```bash +trufflehog huggingface --org my-org --concurrency 10 --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning Discussions and Pull Requests + +```bash +trufflehog huggingface --org my-org --include-discussions --include-prs --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +Only the comments on a discussion or pull request are scanned, there is no separate description text to scan. + +### Self Hosted Endpoint + +```bash +trufflehog huggingface --org my-org --endpoint https://huggingface.mycompany.com --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +## How Scanning Works + +### Scanning Process + +1. **Enumeration**: Lists models, spaces, datasets, and buckets from the given organizations and users, applying the include and exclude filters, then adds any models, spaces, datasets, and buckets given directly +2. **Model, Space, and Dataset Cloning**: Clones each one using the git source, the same way the git source scans a remote repository +3. **Commit Scanning**: Scans every commit in the clone for secrets, the same way the git source does, then deletes the clone +4. **Bucket File Listing**: For each bucket, lists its files, skipping anything larger than 250MB +5. **Bucket File Scanning**: Downloads each remaining bucket file and scans its contents directly, without cloning +6. **Discussion Scanning**: If `--include-discussions` or `--include-prs` is set, fetches discussions and pull requests for each model, space, and dataset and scans their comments +7. **Progress Tracking**: Keeps track of which models, spaces, and datasets have already been scanned, so a scan can be resumed without redoing work + +### What Gets Scanned + +- Every commit in every model, space, and dataset matching the scan target and filters +- Files in every bucket matching the scan target and filters, up to 250MB in size +- Comments on discussions and pull requests, when `--include-discussions` or `--include-prs` is set + +### What Doesn't Get Scanned + +- An entire resource type, when its `--skip-all-*` flag is set +- Models, spaces, datasets, or buckets excluded by an `--include-*` or `--ignore-*` filter, for an organization or user scan +- Bucket files larger than 250MB +- Discussion and pull request descriptions, only their comments are scanned + +## Usage Examples + +### Scanning an Organization + +```bash +trufflehog huggingface --org my-org --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning a Specific Model + +```bash +trufflehog huggingface --model my-org/my-model --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning an Organization's Datasets Only + +```bash +trufflehog huggingface --org my-org --skip-all-models --skip-all-spaces --skip-all-buckets --token hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +### Scanning a Public Model Without Authentication + +```bash +trufflehog huggingface --model my-org/public-model +``` + +## Troubleshooting + +### Common Issues + +**Issue**: `invalid config: you must specify at least one organization, user, model, space, dataset or bucket` +**Solution**: Pass at least one of `--org`, `--user`, `--model`, `--space`, `--dataset`, or `--bucket`. + +--- + +**Issue**: `invalid owner/repo: ` +**Solution**: A value passed to `--include-models`, `--ignore-models`, or any of the other include and ignore flags is not in `owner/name` form. TruffleHog checks this for the include and ignore flags only, not for `--model`, `--space`, `--dataset`, or `--bucket` themselves. + +--- + +**Issue**: `invalid API key` +**Solution**: Check that the token is correct. This also appears if a token is required but was left out. + +--- + +**Issue**: `access to this repo is restricted and you are not in the authorized list` +**Solution**: The authenticated account, or an unauthenticated request, does not have permission to read this model, space, or dataset. Request access on HuggingFace first. diff --git a/pkg/sources/huggingface/bucket.go b/pkg/sources/huggingface/bucket.go new file mode 100644 index 000000000000..54800f90f21b --- /dev/null +++ b/pkg/sources/huggingface/bucket.go @@ -0,0 +1,150 @@ +package huggingface + +import ( + "fmt" + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// maxBucketFileSize is the largest bucket file that will be downloaded and +// scanned. Files larger than this are skipped. +const maxBucketFileSize = 250 * 1024 * 1024 // 250MB + +// cacheBucketInfo fetches bucket metadata (notably its visibility) and caches +// it, then adds the bucket to the list of buckets to scan. Unlike models, +// spaces, and datasets, buckets are not git repos, so they are keyed by their +// plain "namespace/name" ID instead of a clone URL. +func (s *Source) cacheBucketInfo(ctx context.Context, bucketID string) error { + bucketCtx := context.WithValue(ctx, BUCKET, bucketID) + + if _, ok := s.repoInfoCache.get(bucketID); !ok { + bucketCtx.Logger().V(2).Info("Caching bucket info") + bucket, err := s.apiClient.GetBucket(bucketCtx, bucketID) + if err != nil { + bucketCtx.Logger().Error(err, "failed to fetch bucket") + return err + } + if bucket.BucketID == "" { + bucketCtx.Logger().Error(fmt.Errorf("no bucket found for bucket"), bucketID) + return nil + } + s.repoInfoCache.put(bucketID, repoInfo{ + owner: strings.Split(bucket.BucketID, "/")[0], + name: strings.Split(bucket.BucketID, "/")[1], + fullName: bucket.BucketID, + visibility: getVisibility(bucket.IsPrivate), + resourceType: BUCKET, + }) + } + s.buckets = append(s.buckets, bucketID) + return nil +} + +// fetchAndCacheBuckets enumerates all buckets belonging to an author (user or +// org) and caches the ones that survive include/exclude filtering. +func (s *Source) fetchAndCacheBuckets(ctx context.Context, author string) error { + buckets, err := s.apiClient.ListBucketsByAuthor(ctx, author) + if err != nil { + return err + } + for _, bucket := range buckets { + s.filteredBucketsCache.Set(bucket.BucketID, bucket.BucketID) + // Set is a no-op for buckets excluded by include/ignore globs. + if _, ok := s.filteredBucketsCache.Get(bucket.BucketID); !ok { + continue + } + if err := s.cacheBucketInfo(ctx, bucket.BucketID); err != nil { + continue + } + } + return nil +} + +// scanBuckets lists and scans the files of all enumerated buckets. Buckets +// are object storage rather than git repos, so files are downloaded +// individually and chunked directly. +func (s *Source) scanBuckets(ctx context.Context, chunksChan chan *sources.Chunk) error { + ctx.Logger().V(2).Info("Found buckets to scan", "count", len(s.buckets)) + + scanErrs := sources.NewScanErrors() + reporter := sources.ChanReporter{Ch: chunksChan} + + for _, bucketID := range s.buckets { + bucketInfo, ok := s.repoInfoCache.get(bucketID) + if !ok { + // This should never happen. + err := fmt.Errorf("no bucketInfo for bucket: %s", bucketID) + ctx.Logger().Error(err, "failed to scan bucket") + continue + } + bucketCtx := context.WithValues(ctx, BUCKET, bucketID) + + files, err := s.apiClient.ListBucketFiles(bucketCtx, bucketID) + if err != nil { + scanErrs.Add(fmt.Errorf("error listing files in bucket %s: %w", bucketID, err)) + continue + } + + for _, file := range files { + s.jobPool.Go(func() error { + if common.IsDone(bucketCtx) { + return nil + } + if file.Type != "file" { + return nil + } + if file.Size > maxBucketFileSize { + bucketCtx.Logger().V(2).Info("Skipping bucket file: exceeds max size", "path", file.Path, "size", file.Size) + return nil + } + if err := s.scanBucketFile(bucketCtx, bucketID, bucketInfo, file, reporter); err != nil { + scanErrs.Add(fmt.Errorf("error scanning file %s in bucket %s: %w", file.Path, bucketID, err)) + } + return nil + }) + } + } + + _ = s.jobPool.Wait() + if scanErrs.Count() > 0 { + ctx.Logger().V(0).Info("failed to scan some buckets", "error_count", scanErrs.Count(), "errors", scanErrs.String()) + } + s.SetProgressComplete(len(s.buckets), len(s.buckets), "Completed HuggingFace bucket scan", "") + return nil +} + +// scanBucketFile downloads a single bucket file and chunks its content. +func (s *Source) scanBucketFile(ctx context.Context, bucketID string, bucketInfo repoInfo, file BucketFile, reporter sources.ChanReporter) error { + body, err := s.apiClient.DownloadBucketFile(ctx, bucketID, file.Path) + if err != nil { + return err + } + defer func() { _ = body.Close() }() + + chunkSkel := &sources.Chunk{ + SourceType: s.Type(), + SourceName: s.name, + SourceID: s.SourceID(), + JobID: s.JobID(), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Huggingface{ + Huggingface: &source_metadatapb.Huggingface{ + File: sanitizer.UTF8(file.Path), + Link: sanitizer.UTF8(fmt.Sprintf("%s/%s/%s/resolve/%s", s.conn.Endpoint, BucketsRoute, bucketID, escapePathSegments(file.Path))), + Repository: sanitizer.UTF8(fmt.Sprintf("%s/%s/%s", s.conn.Endpoint, BucketsRoute, bucketID)), + Visibility: bucketInfo.visibility, + ResourceType: BUCKET, + }, + }, + }, + SourceVerify: s.verify, + } + + return handlers.HandleFile(ctx, body, chunkSkel, reporter) +} diff --git a/pkg/sources/huggingface/client.go b/pkg/sources/huggingface/client.go index ecfdf093ac3f..3a6b5f6ba491 100644 --- a/pkg/sources/huggingface/client.go +++ b/pkg/sources/huggingface/client.go @@ -4,7 +4,10 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" + "net/url" + "strings" "time" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -16,12 +19,14 @@ var apiPaths = map[string]string{ DATASET: DatasetsRoute, MODEL: ModelsAPIRoute, SPACE: SpacesRoute, + BUCKET: BucketsRoute, } var htmlPaths = map[string]string{ DATASET: DatasetsRoute, MODEL: "", SPACE: SpacesRoute, + BUCKET: BucketsRoute, } type Author struct { @@ -114,14 +119,43 @@ type Repo struct { RepoID string `json:"id"` } +type Bucket struct { + IsPrivate bool `json:"private"` + BucketID string `json:"id"` +} + +type BucketFile struct { + Type string `json:"type"` + Path string `json:"path"` + Size int64 `json:"size"` +} + type HFClient struct { BaseURL string APIKey string HTTPClient *http.Client + // downloadClient streams file bodies. Unlike HTTPClient it has no overall + // Timeout, because http.Client.Timeout also interrupts Response.Body reads — + // which would kill large/slow downloads handed to handlers.HandleFile. + downloadClient *http.Client } // NewHFClient creates a new HF client func NewHFClient(baseURL, apiKey string, timeout time.Duration) *HFClient { + // Bound the connect + response-header phase (including redirect hops to the + // CDN) without capping the body read; overall cancellation comes from the + // request context. Built explicitly (rather than cloning DefaultTransport) + // so it stays a *http.Transport even when tests swap the default transport. + downloadTransport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: timeout, + } + return &HFClient{ BaseURL: baseURL, APIKey: apiKey, @@ -129,6 +163,9 @@ func NewHFClient(baseURL, apiKey string, timeout time.Duration) *HFClient { Timeout: timeout, Transport: common.NewInstrumentedTransport(common.NewCustomTransport(nil)), }, + downloadClient: &http.Client{ + Transport: downloadTransport, + }, } } @@ -159,7 +196,7 @@ func (c *HFClient) get(ctx context.Context, url string, target interface{}) erro return errors.New("access to this repo is restricted and you are not in the authorized list. Visit the repository to ask for access") } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() return json.NewDecoder(resp.Body).Decode(target) } @@ -207,6 +244,136 @@ func (c *HFClient) ListReposByAuthor(ctx context.Context, resourceType string, a return repos, err } +// GetBucket retrieves bucket metadata from the Hugging Face API. +func (c *HFClient) GetBucket(ctx context.Context, bucketID string) (Bucket, error) { + var bucket Bucket + url, err := buildAPIURL(c.BaseURL, BUCKET, bucketID) + if err != nil { + return bucket, err + } + err = c.get(ctx, url, &bucket) + return bucket, err +} + +// ListBucketsByAuthor retrieves buckets from the Hugging Face API by author (user or org). +func (c *HFClient) ListBucketsByAuthor(ctx context.Context, author string) ([]Bucket, error) { + var buckets []Bucket + url := fmt.Sprintf("%s/%s/%s/%s?limit=%d", c.BaseURL, APIRoute, BucketsRoute, author, bucketsPagination) + for url != "" { + var page []Bucket + next, err := c.getPage(ctx, url, &page) + if err != nil { + return buckets, err + } + buckets = append(buckets, page...) + url = next + } + return buckets, nil +} + +// ListBucketFiles recursively lists all files in a bucket, following Link +// header pagination. +func (c *HFClient) ListBucketFiles(ctx context.Context, bucketID string) ([]BucketFile, error) { + var files []BucketFile + url := fmt.Sprintf("%s/%s/%s/%s/tree?recursive=true&limit=%d", c.BaseURL, APIRoute, BucketsRoute, bucketID, bucketsPagination) + for url != "" { + var page []BucketFile + next, err := c.getPage(ctx, url, &page) + if err != nil { + return files, err + } + files = append(files, page...) + url = next + } + return files, nil +} + +// DownloadBucketFile streams a file's content from a bucket via the resolve +// endpoint. The caller is responsible for closing the returned reader. +func (c *HFClient) DownloadBucketFile(ctx context.Context, bucketID string, path string) (io.ReadCloser, error) { + // Escape each path segment individually so special characters (spaces, + // '?', '#', etc.) are encoded while slashes are preserved as separators. + downloadURL := fmt.Sprintf("%s/%s/%s/resolve/%s", c.BaseURL, BucketsRoute, bucketID, escapePathSegments(path)) + + req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create HuggingFace bucket download request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.APIKey) + + resp, err := c.downloadClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download bucket file %s: %w", path, err) + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, fmt.Errorf("failed to download bucket file %s: status %d", path, resp.StatusCode) + } + return resp.Body, nil +} + +// getPage makes a GET request like get, but additionally returns the "next" +// URL from the response Link header, if any, for paginated endpoints. +func (c *HFClient) getPage(ctx context.Context, url string, target interface{}) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("failed to create HuggingFace API request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.APIKey) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to make request to HuggingFace API: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusUnauthorized { + return "", errors.New("invalid API key") + } + if resp.StatusCode == http.StatusForbidden { + return "", errors.New("access to this resource is restricted and you are not in the authorized list. Visit the resource to ask for access") + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status %d from HuggingFace API", resp.StatusCode) + } + + if err := json.NewDecoder(resp.Body).Decode(target); err != nil { + return "", err + } + return parseNextLink(resp.Header.Get("Link")), nil +} + +// escapePathSegments URL-encodes each slash-separated segment of a path while +// keeping the slashes as separators, e.g. "a b/c?d" -> "a%20b/c%3Fd". +func escapePathSegments(path string) string { + segments := strings.Split(path, "/") + for i, segment := range segments { + segments[i] = url.PathEscape(segment) + } + return strings.Join(segments, "/") +} + +// parseNextLink extracts the URL with rel="next" from a Link header value. +// Example: ; rel="next" +func parseNextLink(header string) string { + for _, part := range strings.Split(header, ",") { + sections := strings.Split(part, ";") + if len(sections) < 2 { + continue + } + urlPart := strings.Trim(strings.TrimSpace(sections[0]), "<>") + for _, param := range sections[1:] { + if strings.TrimSpace(param) == `rel="next"` { + return urlPart + } + } + } + return "" +} + // getResourceAPIPath returns the API path for the given resource type func getResourceAPIPath(resourceType string) string { return apiPaths[resourceType] diff --git a/pkg/sources/huggingface/huggingface.go b/pkg/sources/huggingface/huggingface.go index 133510020eab..996a86ca3f08 100644 --- a/pkg/sources/huggingface/huggingface.go +++ b/pkg/sources/huggingface/huggingface.go @@ -32,12 +32,18 @@ const ( DatasetsRoute = "datasets" SpacesRoute = "spaces" ModelsAPIRoute = "models" + BucketsRoute = "buckets" DiscussionsRoute = "discussions" APIRoute = "api" DATASET = "dataset" MODEL = "model" SPACE = "space" + BUCKET = "bucket" defaultPagination = 100 + + // bucketsPagination is the page size for bucket listing endpoints, + // which default to 100 items per page (5000 max). + bucketsPagination = 5000 ) type resourceType string @@ -56,10 +62,12 @@ type Source struct { models []string spaces []string datasets []string + buckets []string filteredModelsCache *filteredRepoCache filteredSpacesCache *filteredRepoCache filteredDatasetsCache *filteredRepoCache + filteredBucketsCache *filteredRepoCache repoInfoCache repoInfoCache @@ -76,6 +84,7 @@ type Source struct { skipAllModels bool skipAllSpaces bool skipAllDatasets bool + skipAllBuckets bool includeDiscussions bool includePrs bool @@ -220,9 +229,16 @@ func (s *Source) Init(ctx context.Context, name string, jobID sources.JobID, sou s.conn.GetIgnoreDatasets(), ) + s.filteredBucketsCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](), + append(s.conn.GetBuckets(), s.conn.GetIncludeBuckets()...), + s.conn.GetIgnoreBuckets(), + ) + s.models = initializeRepos(s.filteredModelsCache, s.conn.Models, fmt.Sprintf("%s/%s.git", s.conn.Endpoint, "%s")) s.spaces = initializeRepos(s.filteredSpacesCache, s.conn.Spaces, fmt.Sprintf("%s/%s/%s.git", s.conn.Endpoint, SpacesRoute, "%s")) s.datasets = initializeRepos(s.filteredDatasetsCache, s.conn.Datasets, fmt.Sprintf("%s/%s/%s.git", s.conn.Endpoint, DatasetsRoute, "%s")) + // Buckets are not git repos; they are keyed by their plain "namespace/name" ID. + s.buckets = initializeRepos(s.filteredBucketsCache, s.conn.Buckets, "%s") s.repoInfoCache = newRepoInfoCache() s.includeDiscussions = s.conn.IncludeDiscussions @@ -262,6 +278,7 @@ func (s *Source) Init(ctx context.Context, name string, jobID sources.JobID, sou s.skipAllModels = s.conn.SkipAllModels s.skipAllSpaces = s.conn.SkipAllSpaces s.skipAllDatasets = s.conn.SkipAllDatasets + s.skipAllBuckets = s.conn.SkipAllBuckets return nil } @@ -285,6 +302,12 @@ func (s *Source) validateIgnoreIncludeRepos() error { if err := verifySlashSeparatedStrings(s.conn.IncludeDatasets); err != nil { return err } + if err := verifySlashSeparatedStrings(s.conn.IgnoreBuckets); err != nil { + return err + } + if err := verifySlashSeparatedStrings(s.conn.IncludeBuckets); err != nil { + return err + } return nil } @@ -366,12 +389,20 @@ func (s *Source) enumerate(ctx context.Context) error { } } - ctx.Logger().Info("Completed enumeration", "num_models", len(s.models), "num_spaces", len(s.spaces), "num_datasets", len(s.datasets)) + s.buckets = make([]string, 0, s.filteredBucketsCache.Count()) + for _, bucket := range s.filteredBucketsCache.Keys() { + if err := s.cacheBucketInfo(ctx, bucket); err != nil { + continue + } + } + + ctx.Logger().Info("Completed enumeration", "num_models", len(s.models), "num_spaces", len(s.spaces), "num_datasets", len(s.datasets), "num_buckets", len(s.buckets)) // We must sort the repos so we can resume later if necessary. sort.Strings(s.models) sort.Strings(s.datasets) sort.Strings(s.spaces) + sort.Strings(s.buckets) return nil } @@ -475,6 +506,12 @@ func (s *Source) enumerateAuthors(ctx context.Context) { continue } } + if !s.skipAllBuckets { + if err := s.fetchAndCacheBuckets(orgCtx, org); err != nil { + orgCtx.Logger().Error(err, "Failed to fetch buckets for organization") + continue + } + } } for _, user := range s.usersCache.Keys() { userCtx := context.WithValue(ctx, "user", user) @@ -496,6 +533,12 @@ func (s *Source) enumerateAuthors(ctx context.Context) { continue } } + if !s.skipAllBuckets { + if err := s.fetchAndCacheBuckets(userCtx, user); err != nil { + userCtx.Logger().Error(err, "Failed to fetch buckets for user") + continue + } + } } } @@ -590,6 +633,9 @@ func (s *Source) scan(ctx context.Context, chunksChan chan *sources.Chunk) error if err := s.scanRepos(ctx, chunksChan, DATASET); err != nil { return err } + if err := s.scanBuckets(ctx, chunksChan); err != nil { + return err + } return nil } @@ -599,7 +645,7 @@ func (s *Source) cloneAndScanRepo(ctx context.Context, repoURL string, repoInfo if err != nil { return 0, err } - defer os.RemoveAll(path) + defer func() { _ = os.RemoveAll(path) }() var logger logr.Logger logger.V(2).Info("scanning %s", repoInfo.resourceType) diff --git a/pkg/sources/huggingface/huggingface_client_test.go b/pkg/sources/huggingface/huggingface_client_test.go index 15977d10e653..4a0229066369 100644 --- a/pkg/sources/huggingface/huggingface_client_test.go +++ b/pkg/sources/huggingface/huggingface_client_test.go @@ -2,6 +2,7 @@ package huggingface import ( "fmt" + "net/http" "strconv" "testing" "time" @@ -649,3 +650,40 @@ func TestGetGitPath_SpaceResource(t *testing.T) { path := discussion.GetGitPath() assert.Equal(t, expectedPath, path) } + +func TestNewHFClient_DownloadClientHasNoOverallTimeout(t *testing.T) { + client := NewHFClient("https://huggingface.co", TEST_TOKEN, 10*time.Second) + + // The API client keeps an overall timeout (its body is consumed in-call). + assert.Equal(t, 10*time.Second, client.HTTPClient.Timeout) + + // The download client must NOT have an overall timeout, since that would + // also interrupt streaming Response.Body reads in handlers.HandleFile. + assert.Equal(t, time.Duration(0), client.downloadClient.Timeout) + + // The header phase is still bounded at the transport level. + transport, ok := client.downloadClient.Transport.(*http.Transport) + assert.True(t, ok) + assert.Equal(t, 10*time.Second, transport.ResponseHeaderTimeout) +} + +func TestEscapePathSegments(t *testing.T) { + tests := []struct { + name string + path string + expected string + }{ + {"simple", "file.jsonl", "file.jsonl"}, + {"nested preserves slashes", "synthtraces/abf-123.jsonl", "synthtraces/abf-123.jsonl"}, + {"space", "my dir/my file.jsonl", "my%20dir/my%20file.jsonl"}, + {"question mark", "data?.jsonl", "data%3F.jsonl"}, + {"hash", "v#1/data.jsonl", "v%231/data.jsonl"}, + {"percent", "100%/data.jsonl", "100%25/data.jsonl"}, + {"empty", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, escapePathSegments(tt.path)) + }) + } +} diff --git a/pkg/sources/jenkins/README.md b/pkg/sources/jenkins/README.md new file mode 100644 index 000000000000..4238e7d4fd50 --- /dev/null +++ b/pkg/sources/jenkins/README.md @@ -0,0 +1,177 @@ +# Jenkins Source + +## Overview + +The Jenkins source lets TruffleHog scan the console output of Jenkins build logs for secrets, credentials, and sensitive data. It walks a Jenkins instance's jobs and folders through the Jenkins API, then downloads and scans the build log of every build it finds. + +## Jenkins Fundamentals + +### What does this source scan? + +This source asks the Jenkins API for the list of jobs and folders at a given Jenkins URL, walks into folders recursively, and for every job it finds, downloads the plain text console output of every build and scans it. + +### Key Jenkins Terminology + +| Term | Description | +|------|-------------| +| **Job** | A single configured build task in Jenkins, such as a Freestyle project or a Pipeline | +| **Folder** | A container that groups jobs and can hold other folders | +| **Build** | One run of a job, numbered in order | +| **Console Output** | The full text log produced while a build ran, available as a plain text page | + +## Features + +- **Folder Traversal**: Recursively walks Jenkins folders to find every job underneath them +- **Multiple Authentication Methods**: Username and password, a custom header, or unauthenticated +- **Self Signed Certificate Support**: Can skip TLS certificate verification for instances using one + +## Configuration + +### Authentication Methods + +#### 1. Username and Password + +**CLI Usage:** +```bash +trufflehog jenkins --url https://jenkins.example.com --username my-username --password my-password +``` + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Jenkins + endpoint: https://jenkins.example.com + basic_auth: + username: my-username + password: my-password + name: jenkins-scan + type: SOURCE_TYPE_JENKINS + verify: true +``` + +On the CLI, the URL, username, and password can also be set with the `JENKINS_URL`, `JENKINS_USERNAME`, and `JENKINS_PASSWORD` environment variables instead of the matching flags. This does not apply to a YAML config file, since TruffleHog does not read environment variables while loading YAML. + +--- + +#### 2. Custom Header + +Sends a fixed header, such as an API token header, with every request. This method is only available in the YAML config, not the CLI. + +**YAML Configuration:** +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Jenkins + endpoint: https://jenkins.example.com + header: + key: "Authorization" + value: "Bearer xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + name: jenkins-scan + type: SOURCE_TYPE_JENKINS + verify: true +``` + +--- + +#### 3. Unauthenticated + +**CLI Usage:** + +Leave out `--username` and `--password` to scan without authentication. This is also what happens if only one of the two is given, since the CLI only uses basic auth when both are set together. + +```bash +trufflehog jenkins --url https://jenkins.example.com +``` + +**YAML Configuration:** + +In the YAML config, this must be set explicitly. Leaving out the credential entirely is rejected. + +```yaml +sources: +- connection: + '@type': type.googleapis.com/sources.Jenkins + endpoint: https://jenkins.example.com + unauthenticated: {} + name: jenkins-scan + type: SOURCE_TYPE_JENKINS + verify: true +``` + +### Self Signed Certificates + +```bash +trufflehog jenkins --url https://jenkins.example.com --username my-username --password my-password --insecure-skip-verify-tls +``` + +On the CLI, this can also be set with the `JENKINS_INSECURE_SKIP_VERIFY_TLS` environment variable. + +## How Scanning Works + +### Scanning Process + +1. **Job Listing**: Asks the Jenkins API for the jobs directly under the given URL +2. **Folder Traversal**: For every folder found, repeats the job listing inside that folder, continuing until every nested folder has been walked +3. **Build Listing**: For every job that is a Freestyle project or a Pipeline, lists all of its builds +4. **Build Log Scanning**: Downloads the console output of every build and scans its text for secrets +5. **Progress Tracking**: Reports progress as each job is processed, so a scan can be monitored while it runs + +### What Gets Scanned + +- The console output of every build of every Freestyle project and Pipeline job found underneath the given URL, including jobs inside nested folders + +### What Doesn't Get Scanned + +- Job types other than a Freestyle project or a Pipeline, such as a Multi-configuration project (in the Jenkins API this is the `hudson.matrix.MatrixProject` class) +- Anything whose Jenkins API class is not recognized as a plain folder or as one of the two scannable job types, including any job container that is not the plain Jenkins folder type +- Anything outside the build's console output, such as artifacts, build parameters, or job configuration + +There is no concurrency for this source: jobs, builds, and their console output are all fetched and scanned one at a time, and there is no flag to change that. + +## Usage Examples + +### Scanning with Username and Password + +```bash +trufflehog jenkins --url https://jenkins.example.com --username my-username --password my-password +``` + +### Scanning Without Authentication + +```bash +trufflehog jenkins --url https://jenkins.example.com +``` + +### Scanning an Instance with a Self Signed Certificate + +```bash +trufflehog jenkins --url https://jenkins.example.com --username my-username --password my-password --insecure-skip-verify-tls +``` + +## Troubleshooting + +### Common Issues + +**Issue**: Authentication failures when connecting to Jenkins +**Solution**: Check that the username and password, or the custom header, are correct and have permission to read the target jobs and builds. + +--- + +**Issue**: A username was given but the scan still ran unauthenticated +**Solution**: On the CLI, both `--username` and `--password` must be set together for basic authentication to be used. If either one is missing, the scan falls back to unauthenticated without an error. + +--- + +**Issue**: `Received non-200 status from get jenkins jobs request` +**Solution**: Check the Jenkins URL is correct and reachable, and that the credentials have permission to list jobs at that URL. + +--- + +**Issue**: Certificate errors when connecting to a self managed Jenkins instance +**Solution**: Pass `--insecure-skip-verify-tls` if the instance uses a self signed certificate. + +--- + +**Issue**: Some jobs are missing from the scan +**Solution**: TruffleHog only scans Freestyle projects and Pipeline jobs, and only recurses into the plain Jenkins folder type. A job of another type, or one nested inside a different kind of container, is silently skipped. diff --git a/pkg/sources/jenkins/jenkins.go b/pkg/sources/jenkins/jenkins.go index e51805555d24..73f60eb67955 100644 --- a/pkg/sources/jenkins/jenkins.go +++ b/pkg/sources/jenkins/jenkins.go @@ -1,6 +1,7 @@ package jenkins import ( + "encoding/base64" "encoding/json" "fmt" "io" @@ -46,8 +47,14 @@ type header struct { value string } -// Ensure the Source satisfies the interface at compile time -var _ sources.Source = (*Source)(nil) +// Ensure the Source satisfies the interfaces at compile time. +// SourceUnitChunker is deliberately not implemented yet, which keeps +// SourceManager on the non-unit Chunks path for scans. +var ( + _ sources.Source = (*Source)(nil) + _ sources.SourceUnitEnumerator = (*Source)(nil) + _ sources.SourceUnitUnmarshaller = (*Source)(nil) +) // Type returns the type of source. // It is used for matching source types in configuration and job input. @@ -193,42 +200,93 @@ func (s *Source) GetJenkinsJobs(ctx context.Context) (JenkinsJobResponse, error) } func (s *Source) RecursivelyGetJenkinsObjectsForPath(ctx context.Context, absolutePath string) (JenkinsJobResponse, error) { + collector := new(jobCollector) + err := s.walkJobs(ctx, absolutePath, collector) + return JenkinsJobResponse{Jobs: collector.jobs}, err +} + +// jobCollector is the UnitReporter the non-unit path uses to gather the walk's +// jobs into a slice. UnitErr returns the error it is given, which aborts the +// walk, preserving the fail-fast behavior callers of GetJenkinsJobs have always +// had. +type jobCollector struct { + jobs []JenkinsJob +} + +var _ sources.UnitReporter = (*jobCollector)(nil) + +func (c *jobCollector) UnitOk(_ context.Context, unit sources.SourceUnit) error { + job, ok := unit.(JenkinsJob) + if !ok { + return fmt.Errorf("expected JenkinsJob, got %T", unit) + } + c.jobs = append(c.jobs, job) + return nil +} + +func (c *jobCollector) UnitErr(_ context.Context, err error) error { return err } + +// walkJobs traverses the Jenkins object tree beneath absolutePath, reporting +// every scannable job it finds to the UnitReporter. Recursion continues unless +// the reporter returns an error, which aborts the walk and is returned to the +// caller. +func (s *Source) walkJobs(ctx context.Context, absolutePath string, reporter sources.UnitReporter) error { ctx.Logger().V(3).Info("getting objects", "path", absolutePath) - jobs := JenkinsJobResponse{} objects, err := s.GetJenkinsObjectsForPath(ctx, absolutePath) if err != nil { - return jobs, err + return reporter.UnitErr(ctx, errors.WrapPrefix(err, fmt.Sprintf("failed to get Jenkins objects for path %q", absolutePath), 0)) } ctx.Logger().V(3).Info("got objects", "path", absolutePath, "count", len(objects.Jobs)) for _, job := range objects.Jobs { - ctx.Logger().V(3).Info("processing object", + if common.IsDone(ctx) { + return ctx.Err() + } + + ctx := context.WithValues(ctx, "object_name", job.Name, "object_class", job.Class, "object_url", job.Url) + ctx.Logger().V(3).Info("processing object") if job.Class == "com.cloudbees.hudson.plugins.folder.Folder" { u, err := url.Parse(job.Url) if err != nil { - return jobs, err + if err := reporter.UnitErr(ctx, fmt.Errorf("failed to parse folder URL %q: %w", job.Url, err)); err != nil { + return err + } + continue } - objects, err := s.RecursivelyGetJenkinsObjectsForPath(ctx, u.Path) - if err != nil { - return jobs, err - } - jobs.Jobs = append(jobs.Jobs, objects.Jobs...) - } else { - if job.Class == "hudson.model.FreeStyleProject" || - job.Class == "org.jenkinsci.plugins.workflow.job.WorkflowJob" { - jobs.Jobs = append(jobs.Jobs, job) + if err := s.walkJobs(ctx, u.Path, reporter); err != nil { + return err } + continue + } + + if job.Class != "hudson.model.FreeStyleProject" && + job.Class != "org.jenkinsci.plugins.workflow.job.WorkflowJob" { + continue + } + + parsedUrl, err := url.Parse(job.Url) + if err != nil { + // Skipped rather than reported, so that Enumerate and the walk + // Chunks uses agree on the set of jobs. Chunks skips a job it + // cannot parse a URL for too. + ctx.Logger().Error(err, "failed to parse job URL; skipping job") + continue + } + job.Path = strings.TrimSuffix(parsedUrl.Path, "/") + + if err := reporter.UnitOk(ctx, job); err != nil { + return err } } - return jobs, nil + return nil } func (s *Source) GetJenkinsObjectsForPath(ctx context.Context, absolutePath string) (JenkinsJobResponse, error) { @@ -250,7 +308,7 @@ func (s *Source) GetJenkinsObjectsForPath(ctx context.Context, absolutePath stri if err != nil { return res, errors.WrapPrefix(err, "Failed to do get jenkins jobs request", 0) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return res, errors.New(fmt.Sprintf("Received non-200 status from get jenkins jobs request: %d", resp.StatusCode)) @@ -297,7 +355,7 @@ func (s *Source) GetJenkinsBuilds(ctx context.Context, jobAbsolutePath string) ( if err != nil { return builds, errors.WrapPrefix(err, "Failed to do get jenkins builds request", 0) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return builds, errors.New(fmt.Sprintf("Received non-200 status from get jenkins builds request: %d", resp.StatusCode)) @@ -341,13 +399,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . s.SetProgressComplete(i, len(jobs.Jobs), fmt.Sprintf("Project: %s", project.Name), "") - parsedUrl, err := url.Parse(project.Url) - if err != nil { - ctx.Logger().Error(err, "failed to parse job URL; skipping job") - continue - } + // Path is derived from the job URL by the walk above. projectURL := *s.url - projectURL.Path = parsedUrl.Path + projectURL.Path = project.Path builds, err := s.GetJenkinsBuilds(ctx, projectURL.Path) if err != nil { @@ -404,7 +458,7 @@ func (s *Source) chunkBuild( if err != nil { return fmt.Errorf("could not retrieve build log from %q: %w", buildLogURL.String(), err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode >= 400 { return fmt.Errorf("got unexpected HTTP status code %v when trying to retrieve build log from %q", @@ -433,12 +487,46 @@ func (s *Source) chunkBuild( return handlers.HandleFile(ctx, resp.Body, chunkSkel, sources.ChanReporter{Ch: chunksChan}) } +// Enumerate implements the SourceUnitEnumerator interface. It reports one unit +// per Jenkins job, walking folders recursively. A folder that cannot be listed +// is reported to the UnitReporter and the rest of the walk continues. +func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error { + baseUrl := *s.url + return s.walkJobs(ctx, baseUrl.Path, reporter) +} + +// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface. It +// accepts three shapes: the persisted envelope carrying unit_data (round-trips +// the unit exactly), the envelope without unit_data (rebuilt from id), and a +// bare JenkinsJob. +func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) { + var envelope unitEnvelope + if err := json.Unmarshal(data, &envelope); err == nil && envelope.Kind == string(SourceUnitKindJob) { + if envelope.UnitData != "" { + if decoded, err := base64.StdEncoding.DecodeString(envelope.UnitData); err == nil { + var unit JenkinsJob + if json.Unmarshal(decoded, &unit) == nil && unit.Path != "" { + return unit, nil + } + } + } + if envelope.ID != "" { + return JenkinsJob{Path: envelope.ID}, nil + } + } + + var unit JenkinsJob + if err := json.Unmarshal(data, &unit); err != nil { + return nil, err + } + if unit.Path == "" { + return nil, errors.New("not a Jenkins job unit") + } + return unit, nil +} + type JenkinsJobResponse struct { - Jobs []struct { - Class string `json:"_class"` - Name string `json:"name"` - Url string `json:"url"` - } `json:"jobs"` + Jobs []JenkinsJob `json:"jobs"` } type JenkinsBuildResponse struct { diff --git a/pkg/sources/jenkins/jenkins_test.go b/pkg/sources/jenkins/jenkins_test.go index 90461805e584..3598df30caf7 100644 --- a/pkg/sources/jenkins/jenkins_test.go +++ b/pkg/sources/jenkins/jenkins_test.go @@ -58,10 +58,10 @@ func createMockJenkinsServer(jobName string, buildNumber int, logContent string) response := fmt.Sprintf( `{"jobs":[{"_class":"org.jenkinsci.plugins.workflow.job.WorkflowJob",`+ `"name":"%s","url":"%s/job/%s/"}]}`, jobName, server.URL, jobName) - fmt.Fprint(w, response) + _, _ = fmt.Fprint(w, response) } else { w.Header().Set("Content-Type", "application/json") - fmt.Fprint(w, `{"jobs":[]}`) + _, _ = fmt.Fprint(w, `{"jobs":[]}`) } }) @@ -71,10 +71,10 @@ func createMockJenkinsServer(jobName string, buildNumber int, logContent string) w.Header().Set("Content-Type", "application/json") response := fmt.Sprintf( `{"builds":[{"number":%d,"url":"%s/job/%s/%d/"}]}`, buildNumber, server.URL, jobName, buildNumber) - fmt.Fprint(w, response) + _, _ = fmt.Fprint(w, response) } else { w.Header().Set("Content-Type", "application/json") - fmt.Fprint(w, `{"builds":[]}`) + _, _ = fmt.Fprint(w, `{"builds":[]}`) } }) @@ -82,7 +82,7 @@ func createMockJenkinsServer(jobName string, buildNumber int, logContent string) // This is where the test data payload is served to verify chunking behavior. mux.HandleFunc(fmt.Sprintf("/job/%s/%d/consoleText", jobName, buildNumber), func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") - fmt.Fprint(w, logContent) + _, _ = fmt.Fprint(w, logContent) }) return server @@ -224,7 +224,7 @@ func TestJenkinsChunkBuildDirect(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/consoleText") { w.Header().Set("Content-Type", "text/plain") - fmt.Fprint(w, largeLogContent) + _, _ = fmt.Fprint(w, largeLogContent) } else { http.NotFound(w, r) } diff --git a/pkg/sources/jenkins/unit.go b/pkg/sources/jenkins/unit.go new file mode 100644 index 000000000000..62fd15b3a82c --- /dev/null +++ b/pkg/sources/jenkins/unit.go @@ -0,0 +1,69 @@ +package jenkins + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +const SourceUnitKindJob sources.SourceUnitKind = "job" + +// jobPathSegment is the path segment Jenkins puts before every job or folder +// name in a job URL, e.g. /job/folder1/job/build. +const jobPathSegment = "job" + +// JenkinsJob is a job as listed by the Jenkins API, and is the source unit the +// Jenkins source enumerates. +type JenkinsJob struct { + Class string `json:"_class"` + Name string `json:"name"` + Url string `json:"url"` + // Path is the job's path relative to the Jenkins instance. Jenkins reports + // absolute job URLs whose host comes from the instance's own root URL + // setting, which can differ from the endpoint being scanned, so only the + // path is retained as the identity. Chunks does the same when it scans a + // job. Path is not part of the Jenkins API response; walkJobs derives it + // from Url, so it is empty on a value decoded straight from a listing. + Path string `json:"path"` +} + +var _ sources.SourceUnit = JenkinsJob{} + +func (j JenkinsJob) SourceUnitID() (string, sources.SourceUnitKind) { + return j.Path, SourceUnitKindJob +} + +// Display returns the job's folder path without the "job" segments Jenkins +// interleaves, e.g. /jenkins/job/folder1/job/build becomes folder1/build. It is +// derived from Path rather than from Name so that a unit rebuilt from its ID +// alone still renders a name. +func (j JenkinsJob) Display() string { + segments := strings.Split(strings.Trim(j.Path, "/"), "/") + + // Walk in pairs so that a job named "job" and any instance base path + // (/jenkins/) are both handled: a name is only ever the segment directly + // after a "job" segment. + names := make([]string, 0, len(segments)/2) + for i := 0; i < len(segments)-1; i++ { + if segments[i] == jobPathSegment { + names = append(names, segments[i+1]) + i++ + } + } + + if len(names) == 0 { + return j.Path + } + return strings.Join(names, "/") +} + +// unitEnvelope is the JSON shape a host application persists for an +// enumerated unit between an enumerate pass and a later scan pass: the +// SourceUnit proto marshalled with encoding/json, where unit_data carries the +// original unit as base64-encoded bytes. +type unitEnvelope struct { + ID string `json:"id"` + Kind string `json:"kind,omitempty"` + Display string `json:"display,omitempty"` + UnitData string `json:"unit_data,omitempty"` +} diff --git a/pkg/sources/jenkins/unit_test.go b/pkg/sources/jenkins/unit_test.go new file mode 100644 index 000000000000..f768c666468e --- /dev/null +++ b/pkg/sources/jenkins/unit_test.go @@ -0,0 +1,401 @@ +package jenkins + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// testUnitReporter records everything Enumerate reports. +type testUnitReporter struct { + mu sync.Mutex + units []sources.SourceUnit + errs []error +} + +func (r *testUnitReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error { + r.mu.Lock() + defer r.mu.Unlock() + r.units = append(r.units, unit) + return nil +} + +func (r *testUnitReporter) UnitErr(_ context.Context, err error) error { + r.mu.Lock() + defer r.mu.Unlock() + r.errs = append(r.errs, err) + return nil +} + +func (r *testUnitReporter) unitIDs(t *testing.T) []string { + t.Helper() + ids := make([]string, 0, len(r.units)) + for _, unit := range r.units { + id, kind := unit.SourceUnitID() + assert.Equal(t, SourceUnitKindJob, kind, "every Jenkins unit must use the job kind") + ids = append(ids, id) + } + return ids +} + +// createMockJenkinsTreeServer serves a nested job tree. The existing mock in +// jenkins_test.go has no folders, so folder recursion is only covered here. +// +// /job/top-job WorkflowJob +// /job/ignored-class MatrixProject, unsupported, dropped +// /job/folder1/ Folder +// /job/folder1/job/nested-job FreeStyleProject +// /job/folder1/job/folder2/ Folder +// /job/folder1/job/folder2/job/deep-job WorkflowJob +// /job/broken/ Folder whose listing is forbidden +func createMockJenkinsTreeServer(withBrokenFolder bool) *httptest.Server { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + object := func(class, name, path string) string { + return fmt.Sprintf(`{"_class":%q,"name":%q,"url":"%s%s"}`, class, name, server.URL, path) + } + jobsResponse := func(w http.ResponseWriter, r *http.Request, objects ...string) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.URL.RawQuery, "tree=jobs") { + _, _ = fmt.Fprint(w, `{"jobs":[]}`) + return + } + _, _ = fmt.Fprintf(w, `{"jobs":[%s]}`, strings.Join(objects, ",")) + } + + root := []string{ + object("org.jenkinsci.plugins.workflow.job.WorkflowJob", "top-job", "/job/top-job/"), + object("hudson.matrix.MatrixProject", "ignored-class", "/job/ignored-class/"), + object("com.cloudbees.hudson.plugins.folder.Folder", "folder1", "/job/folder1/"), + } + if withBrokenFolder { + root = append(root, object("com.cloudbees.hudson.plugins.folder.Folder", "broken", "/job/broken/")) + } + + mux.HandleFunc("/api/json", func(w http.ResponseWriter, r *http.Request) { + jobsResponse(w, r, root...) + }) + mux.HandleFunc("/job/folder1/api/json", func(w http.ResponseWriter, r *http.Request) { + jobsResponse(w, r, + object("hudson.model.FreeStyleProject", "nested-job", "/job/folder1/job/nested-job/"), + object("com.cloudbees.hudson.plugins.folder.Folder", "folder2", "/job/folder1/job/folder2/")) + }) + mux.HandleFunc("/job/folder1/job/folder2/api/json", func(w http.ResponseWriter, r *http.Request) { + jobsResponse(w, r, + object("org.jenkinsci.plugins.workflow.job.WorkflowJob", "deep-job", "/job/folder1/job/folder2/job/deep-job/")) + }) + // 403 rather than 500 so the client's 5XX retry policy doesn't stall the + // test for the full 90s retry budget. + mux.HandleFunc("/job/broken/api/json", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + }) + + return server +} + +func newTestSource(t *testing.T, ctx context.Context, endpoint, name string) *Source { + t.Helper() + + conn, err := anypb.New(&sourcespb.Jenkins{ + Endpoint: endpoint, + Credential: &sourcespb.Jenkins_BasicAuth{ + BasicAuth: &credentialspb.BasicAuth{ + Username: "testuser", + Password: "testpass", + }, + }, + }) + require.NoError(t, err) + + s := new(Source) + require.NoError(t, s.Init(ctx, name, 0, 1, false, conn, runtime.NumCPU())) + return s +} + +// wantedUnitIDs are the jobs the mock tree exposes, in the order the walk finds +// them. The unsupported MatrixProject and the folders themselves are excluded. +var wantedUnitIDs = []string{ + "/job/top-job", + "/job/folder1/job/nested-job", + "/job/folder1/job/folder2/job/deep-job", +} + +func TestEnumerate(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(false) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-enumerate") + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + + assert.Equal(t, wantedUnitIDs, reporter.unitIDs(t)) + assert.Empty(t, reporter.errs) +} + +func TestEnumerateStableAcrossRuns(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(false) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-stable") + + first := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, first)) + second := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, second)) + + assert.Equal(t, first.unitIDs(t), second.unitIDs(t)) +} + +func TestEnumerateContinuesPastFolderError(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(true) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-folder-error") + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + + // The unreadable folder is reported but does not stop the other jobs from + // being enumerated. + assert.Equal(t, wantedUnitIDs, reporter.unitIDs(t)) + require.Len(t, reporter.errs, 1) + assert.Contains(t, reporter.errs[0].Error(), "/job/broken/") +} + +// TestEnumerateMatchesChunkTraversal guards against Enumerate and the walk +// Chunks uses reporting different sets of jobs. +func TestEnumerateMatchesChunkTraversal(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(false) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-parity") + + jobs, err := s.GetJenkinsJobs(ctx) + require.NoError(t, err) + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + + require.Len(t, reporter.units, len(jobs.Jobs)) + for i, job := range jobs.Jobs { + id, _ := reporter.units[i].SourceUnitID() + assert.Equal(t, job.Path, id, "unit does not match the job Chunks would scan (%q)", job.Url) + assert.True(t, strings.HasSuffix(job.Url, id+"/"), + "job path %q was not derived from the job URL %q", job.Path, job.Url) + } +} + +// TestUnparseableJobURLSkippedByBothPaths pins that a job whose URL cannot be +// parsed drops out of enumeration and out of the walk Chunks uses, rather than +// failing either one. +func TestUnparseableJobURLSkippedByBothPaths(t *testing.T) { + t.Parallel() + + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + + mux.HandleFunc("/api/json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.URL.RawQuery, "tree=jobs") { + _, _ = fmt.Fprint(w, `{"jobs":[]}`) + return + } + // The second URL has an invalid percent escape, which url.Parse rejects. + _, _ = fmt.Fprintf(w, `{"jobs":[`+ + `{"_class":"hudson.model.FreeStyleProject","name":"good","url":"%s/job/good/"},`+ + `{"_class":"hudson.model.FreeStyleProject","name":"bad","url":"%s/job/%%zz/"}`+ + `]}`, server.URL, server.URL) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-bad-job-url") + + reporter := new(testUnitReporter) + require.NoError(t, s.Enumerate(ctx, reporter)) + assert.Equal(t, []string{"/job/good"}, reporter.unitIDs(t)) + assert.Empty(t, reporter.errs) + + jobs, err := s.GetJenkinsJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs.Jobs, 1) + assert.Equal(t, "/job/good", jobs.Jobs[0].Path) +} + +// TestGetJenkinsJobsStillFailsFast pins the pre-existing behavior of the walk +// Chunks relies on: an unreadable folder aborts the traversal. +func TestGetJenkinsJobsStillFailsFast(t *testing.T) { + t.Parallel() + + server := createMockJenkinsTreeServer(true) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := newTestSource(t, ctx, server.URL, "test-jenkins-fail-fast") + + _, err := s.GetJenkinsJobs(ctx) + require.Error(t, err) +} + +func TestJenkinsJobDisplay(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + want string + }{ + {name: "top level", path: "/job/build", want: "build"}, + {name: "nested", path: "/job/folder1/job/sub/job/build", want: "folder1/sub/build"}, + // A folder holding many jobs is the normal case, so the leaf name has + // to survive into the display string or siblings would all render the + // same. See TestJenkinsJobDisplaySiblingsDiffer. + {name: "sibling in same folder", path: "/job/folder1/job/first", want: "folder1/first"}, + {name: "other sibling in same folder", path: "/job/folder1/job/second", want: "folder1/second"}, + {name: "instance base path", path: "/jenkins/job/folder1/job/build", want: "folder1/build"}, + {name: "job named job", path: "/job/job/job/build", want: "job/build"}, + {name: "trailing slash", path: "/job/build/", want: "build"}, + {name: "no job segment falls back", path: "/some/other/path", want: "/some/other/path"}, + {name: "empty falls back", path: "", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, JenkinsJob{Path: tt.path}.Display()) + }) + } +} + +// TestJenkinsJobDisplaySiblingsDiffer pins the property that distinct jobs never +// collapse to the same display string, including jobs sharing a folder. +func TestJenkinsJobDisplaySiblingsDiffer(t *testing.T) { + t.Parallel() + + paths := []string{ + "/job/folder1/job/a", + "/job/folder1/job/b", + "/job/folder1/job/sub/job/a", + "/job/folder2/job/a", + "/job/a", + } + + seen := make(map[string]string, len(paths)) + for _, path := range paths { + display := JenkinsJob{Path: path}.Display() + if previous, ok := seen[display]; ok { + t.Errorf("paths %q and %q both display as %q", previous, path, display) + } + seen[display] = path + } +} + +func TestJenkinsJobSourceUnitID(t *testing.T) { + t.Parallel() + + id, kind := JenkinsJob{Path: "/job/folder1/job/build"}.SourceUnitID() + assert.Equal(t, "/job/folder1/job/build", id) + assert.Equal(t, SourceUnitKindJob, kind) +} + +func TestUnmarshalSourceUnit(t *testing.T) { + t.Parallel() + + const jobPath = "/job/folder1/job/build" + + unitData, err := json.Marshal(JenkinsJob{Path: jobPath}) + require.NoError(t, err) + + envelopeWithData, err := json.Marshal(unitEnvelope{ + ID: jobPath, + Kind: string(SourceUnitKindJob), + Display: "folder1/build", + UnitData: base64.StdEncoding.EncodeToString(unitData), + }) + require.NoError(t, err) + + envelopeIDOnly, err := json.Marshal(unitEnvelope{ + ID: jobPath, + Kind: string(SourceUnitKindJob), + Display: "folder1/build", + }) + require.NoError(t, err) + + tests := []struct { + name string + data []byte + want string + wantErr bool + }{ + {name: "envelope with unit data", data: envelopeWithData, want: jobPath}, + {name: "envelope without unit data", data: envelopeIDOnly, want: jobPath}, + {name: "bare unit", data: unitData, want: jobPath}, + { + name: "envelope with wrong kind", + data: fmt.Appendf(nil, `{"id":%q,"kind":"bucket"}`, jobPath), + wantErr: true, + }, + {name: "empty path", data: []byte(`{"path":""}`), wantErr: true}, + {name: "not json", data: []byte(`not json`), wantErr: true}, + } + + s := new(Source) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + unit, err := s.UnmarshalSourceUnit(tt.data) + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + + id, kind := unit.SourceUnitID() + assert.Equal(t, tt.want, id) + assert.Equal(t, SourceUnitKindJob, kind) + }) + } +} diff --git a/pkg/sources/job_progress.go b/pkg/sources/job_progress.go index e3965e4e67cb..88d6f3a1dd4e 100644 --- a/pkg/sources/job_progress.go +++ b/pkg/sources/job_progress.go @@ -39,6 +39,13 @@ type JobProgressHook interface { Finish(JobProgressRef) } +var ( + // ErrNoJob is returned when there is no JobProgress associated with the reference. + ErrNoJob = errors.New("no job") + // ErrJobDone is returned when the job finishes without being cancelled. + ErrJobDone = errors.New("job done") +) + // JobProgressRef is a wrapper of a JobProgress for read-only access to its state. // If the job supports it, the reference can also be used to cancel running via // CancelRun. @@ -57,6 +64,25 @@ func (r *JobProgressRef) Snapshot() JobProgressMetrics { return r.jobProgress.Snapshot() } +// Err reports why the job ended, or nil if it is still running. See +// JobProgress.Err for the possible values. If the ref does not reference a +// JobProgress, it does not track any job and ErrNoJob is returned. +func (r *JobProgressRef) Err() error { + if r.jobProgress == nil { + return ErrNoJob + } + return r.jobProgress.Err() +} + +// UnitProgress returns the progress the source last reported inside the unit +// with the provided ID, and whether it reported any. +func (r *JobProgressRef) UnitProgress(unitID string) (UnitProgress, bool) { + if r.jobProgress == nil { + return UnitProgress{}, false + } + return r.jobProgress.UnitProgress(unitID) +} + // Done returns a channel that will block until the job has completed. func (r *JobProgressRef) Done() <-chan struct{} { if r.jobProgress == nil { @@ -75,7 +101,15 @@ func (r *JobProgressRef) CancelRun(cause error) { if r.jobProgress == nil || r.jobProgress.jobCancel == nil { return } - r.jobProgress.jobCancel(cause) + if cause == nil { + cause = context.Canceled + } + + r.jobProgress.cancelCauseOnce.Do(func() { + r.jobProgress.cancelCause = cause + }) + // cancel with the first recorded cause. + r.jobProgress.jobCancel(r.jobProgress.cancelCause) } // Fatal is a wrapper around error to differentiate non-fatal errors from fatal @@ -108,11 +142,17 @@ type JobProgress struct { JobID JobID SourceID SourceID SourceName string - // Tracks whether the job is finished or not. + // ctx represents the job's lifetime. It stays open while the job runs and is + // closed by Finish, at which point Done unblocks and Err returns the reason + // the job ended (the CancelRun cause or ErrJobDone). ctx context.Context - cancel context.CancelFunc - // Requests to cancel the job. + cancel context.CancelCauseFunc + // jobCancel stops the source's run context. It requests shutdown without + // marking the job complete. Finish remains responsible for closing Done. jobCancel context.CancelCauseFunc + // Tracks the cause of the cancellation. First cause is used. + cancelCauseOnce sync.Once + cancelCause error // Metrics. metrics JobProgressMetrics metricsLock sync.Mutex @@ -160,7 +200,7 @@ func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) { // NewJobProgress creates a new job report for the given source and job ID. func NewJobProgress(jobID JobID, sourceID SourceID, sourceName string, opts ...func(*JobProgress)) *JobProgress { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancelCause(context.Background()) jp := &JobProgress{ JobID: jobID, SourceID: sourceID, @@ -210,10 +250,31 @@ func (jp *JobProgress) End(end time.Time) { jp.executeHooks(func(hook JobProgressHook) { hook.End(jp.Ref(), end) }) } + +// finishWithCause finishes the job with the given cause. Just calling Finish +// would use ErrJobDone. This would be inaccurate in some cases, for example, if +// a job is rejected by the source manager. +func (jp *JobProgress) finishWithCause(cause error) { + jp.cancelCauseOnce.Do(func() { + jp.cancelCause = cause + }) + jp.Finish() +} + func (jp *JobProgress) Finish() { - jp.cancel() + jp.cancelCauseOnce.Do(func() { + jp.cancelCause = ErrJobDone + }) + jp.cancel(jp.cancelCause) jp.executeHooks(func(hook JobProgressHook) { hook.Finish(jp.Ref()) }) } + +// Err mirrors context.Err for the job. It returns: +// - nil: The job is still running. This includes the period after CancelRun +// is called but before the job finishes. +// - the cause passed to CancelRun: the job finished after being cancelled. +// - ErrJobDone: the job finished without being cancelled. +func (jp *JobProgress) Err() error { return context.Cause(jp.ctx) } func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() } func (jp *JobProgress) ReportUnit(unit SourceUnit) { jp.metricsLock.Lock() @@ -284,6 +345,19 @@ func (jp *JobProgress) Snapshot() JobProgressMetrics { return metrics } +// UnitProgress safely gets the progress the source last reported inside the +// unit with the provided ID. +func (jp *JobProgress) UnitProgress(unitID string) (UnitProgress, bool) { + jp.metricsLock.Lock() + progress := jp.progress + jp.metricsLock.Unlock() + + if progress == nil { + return UnitProgress{}, false + } + return progress.GetUnitProgressFor(unitID) +} + // ReportError adds a non-nil error to the aggregate of errors // encountered during scanning. func (jp *JobProgress) ReportError(err error) { diff --git a/pkg/sources/job_progress_hook.go b/pkg/sources/job_progress_hook.go index 6f94feff276e..973f2b76e905 100644 --- a/pkg/sources/job_progress_hook.go +++ b/pkg/sources/job_progress_hook.go @@ -17,6 +17,13 @@ type UnitHook struct { mu sync.Mutex finishedMetrics chan UnitMetrics logBackPressure func() + // closeMu, closed, done, and inflight coordinate Close with in-flight + // ejectFinishedMetrics sends from still-running job goroutines, so + // finishedMetrics is never closed while a send is possible. + closeMu sync.Mutex + closed bool + done chan struct{} + inflight sync.WaitGroup NoopHook } @@ -36,6 +43,7 @@ func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) (*UnitHook, <-chan Un hook := UnitHook{ metrics: make(map[string]*UnitMetrics, runtime.NumCPU()), finishedMetrics: make(chan UnitMetrics, 1024), + done: make(chan struct{}), logBackPressure: func() { once.Do(func() { ctx.Logger().Info("back pressure detected in unit hook") @@ -71,6 +79,17 @@ func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string { } func (u *UnitHook) ejectFinishedMetrics(metrics UnitMetrics) { + // Register as an in-flight sender so Close waits for us before closing + // the channel; metrics arriving after Close are dropped. + u.closeMu.Lock() + if u.closed { + u.closeMu.Unlock() + return + } + u.inflight.Add(1) + u.closeMu.Unlock() + defer u.inflight.Done() + // Intentionally block the hook from returning to supply back-pressure // to the source. select { @@ -79,7 +98,12 @@ func (u *UnitHook) ejectFinishedMetrics(metrics UnitMetrics) { default: u.logBackPressure() } - u.finishedMetrics <- metrics + select { + case u.finishedMetrics <- metrics: + case <-u.done: + // Close was called with no consumer draining the channel; drop the + // metrics rather than block Close forever. + } } func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) { @@ -189,6 +213,19 @@ func (u *UnitHook) InProgressSnapshot() []UnitMetrics { } func (u *UnitHook) Close() error { + u.closeMu.Lock() + if u.closed { + u.closeMu.Unlock() + return nil + } + u.closed = true + close(u.done) + u.closeMu.Unlock() + + // Wait for in-flight sends before closing the channel; a send on a + // closed channel is a panic, and job goroutines can still be finishing + // units when Close is called. + u.inflight.Wait() close(u.finishedMetrics) return nil } diff --git a/pkg/sources/job_progress_test.go b/pkg/sources/job_progress_test.go index 20ba0678550a..9058c4a72989 100644 --- a/pkg/sources/job_progress_test.go +++ b/pkg/sources/job_progress_test.go @@ -1,6 +1,7 @@ package sources import ( + "context" "errors" "fmt" "testing" @@ -107,6 +108,77 @@ func TestJobProgressHook(t *testing.T) { jp.Finish() } +func TestJobProgressErr(t *testing.T) { + t.Run("ref with no job", func(t *testing.T) { + ref := JobProgressRef{} + assert.ErrorIs(t, ref.Err(), ErrNoJob) + assert.NotErrorIs(t, ref.Err(), ErrJobDone) + }) + + // function to create a cancellable job + newCancellableJob := func() (context.Context, *JobProgress, JobProgressRef) { + runCtx, cancel := context.WithCancelCause(context.Background()) + jp := NewJobProgress(123, 456, "source name", WithCancel(cancel)) + return runCtx, jp, jp.Ref() + } + t.Run("finished without cancel", func(t *testing.T) { + _, jp, ref := newCancellableJob() + // Still running: Err is nil on both the job and the ref. + assert.NoError(t, jp.Err()) + assert.NoError(t, ref.Err()) + jp.Finish() + <-ref.Done() + assert.ErrorIs(t, jp.Err(), ErrJobDone) + assert.ErrorIs(t, ref.Err(), ErrJobDone) + }) + t.Run("cancelled via CancelRun", func(t *testing.T) { + _, jp, ref := newCancellableJob() + cause := fmt.Errorf("abort! abort!") + ref.CancelRun(cause) + // CancelRun only requests a stop. Until Finish runs, Done is still + // open and Err must still report the job as running. + select { + case <-ref.Done(): + assert.FailNow(t, "job should not be done before Finish") + default: + } + assert.NoError(t, jp.Err()) + assert.NoError(t, ref.Err()) + jp.Finish() + <-ref.Done() + assert.ErrorIs(t, jp.Err(), cause) + assert.ErrorIs(t, ref.Err(), cause) + }) + t.Run("cancel after finish does not change Err", func(t *testing.T) { + _, jp, ref := newCancellableJob() + jp.Finish() + ref.CancelRun(fmt.Errorf("too late")) + assert.ErrorIs(t, ref.Err(), ErrJobDone) + }) + + t.Run("nil cause is reported as cancelled", func(t *testing.T) { + _, jp, ref := newCancellableJob() + ref.CancelRun(nil) + jp.Finish() + assert.ErrorIs(t, ref.Err(), context.Canceled) + assert.NotErrorIs(t, ref.Err(), ErrJobDone) + }) + + t.Run("first cause wins", func(t *testing.T) { + runCtx, jp, ref := newCancellableJob() + first := fmt.Errorf("first cause") + second := fmt.Errorf("second cause") + ref.CancelRun(first) + ref.CancelRun(second) + // The source's run context sees the first cause, not the second. + assert.ErrorIs(t, context.Cause(runCtx), first) + jp.Finish() + <-ref.Done() + assert.ErrorIs(t, ref.Err(), first) + assert.NotErrorIs(t, ref.Err(), second) + }) +} + func TestJobProgressDone(t *testing.T) { ref := JobProgressRef{} select { @@ -151,3 +223,26 @@ func TestJobProgressErrorsFor(t *testing.T) { assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"}))) assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"}))) } + +func TestJobProgressRefUnitProgress(t *testing.T) { + var progress Progress + jp := NewJobProgress(123, 456, "source name") + jp.TrackProgress(&progress) + ref := jp.Ref() + + _, ok := ref.UnitProgress("unit") + assert.False(t, ok) + + want := UnitProgress{State: UnitProgressCounting, ItemsDone: 3, BytesDone: 300} + progress.SetUnitProgressFor("unit", want) + got, ok := ref.UnitProgress("unit") + assert.True(t, ok) + assert.Equal(t, want, got) + + // A ref to no job, or to a job whose source reports no progress, has none to give. + _, ok = (&JobProgressRef{}).UnitProgress("unit") + assert.False(t, ok) + untracked := NewJobProgress(123, 456, "source name").Ref() + _, ok = untracked.UnitProgress("unit") + assert.False(t, ok) +} diff --git a/pkg/sources/json_enumerator/json_enumerator.go b/pkg/sources/json_enumerator/json_enumerator.go index a82cea96d131..54e98dd51f2b 100644 --- a/pkg/sources/json_enumerator/json_enumerator.go +++ b/pkg/sources/json_enumerator/json_enumerator.go @@ -7,8 +7,10 @@ import ( "fmt" "io" "os" + "strings" "unicode/utf8" + "github.com/klauspost/compress/zstd" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -35,6 +37,7 @@ type Source struct { // Ensure the Source satisfies the interfaces at compile time var _ sources.Source = (*Source)(nil) var _ sources.SourceUnitUnmarshaller = (*Source)(nil) +var _ sources.SourceUnitEnumChunker = (*Source)(nil) func (s *Source) Type() sourcespb.SourceType { return SourceType } func (s *Source) SourceID() sources.SourceID { return s.sourceId } @@ -73,7 +76,7 @@ func (s *Source) Chunks( return nil } s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "") - if err := s.chunkJSONEnumerator(ctx, path, chunksChan); err != nil { + if err := s.chunkJSONEnumerator(ctx, path, sources.ChanReporter{Ch: chunksChan}); err != nil { ctx.Logger().Error(err, "error scanning JSON enumerator", "path", path) } } @@ -81,6 +84,34 @@ func (s *Source) Chunks( return nil } +// Enumerate implements the SourceUnitEnumerator interface, reporting each +// configured path as its own unit so paths can be chunked concurrently. +func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error { + for _, path := range s.paths { + f, err := os.Open(path) + if err != nil { + if err := reporter.UnitErr(ctx, err); err != nil { + return err + } + } else { + _ = f.Close() + if err := reporter.UnitOk(ctx, sources.CommonSourceUnit{ID: path}); err != nil { + return err + } + } + } + return nil +} + +// ChunkUnit implements the SourceUnitChunker interface. +func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error { + path, _ := unit.SourceUnitID() + if err := s.chunkJSONEnumerator(ctx, path, reporter); err != nil { + return reporter.ChunkErr(ctx, err) + } + return nil +} + type jsonEntry struct { Metadata json.RawMessage Data []byte @@ -142,13 +173,11 @@ func (e *jsonEntry) UnmarshalJSON(data []byte) error { func (s *Source) chunkJSONEnumeratorReader( ctx context.Context, input io.Reader, - chunksChan chan *sources.Chunk, + reporter sources.ChunkReporter, ) error { decoder := json.NewDecoder(input) var entry jsonEntry - reporter := sources.ChanReporter{Ch: chunksChan} - for { if err := decoder.Decode(&entry); err != nil { if errors.Is(err, io.EOF) { @@ -190,7 +219,7 @@ func (s *Source) chunkJSONEnumeratorReader( func (s *Source) chunkJSONEnumerator( ctx context.Context, path string, - chunksChan chan *sources.Chunk, + reporter sources.ChunkReporter, ) error { ctx.Logger().V(3).Info("chunking JSON enumerator", "path", path) @@ -198,7 +227,17 @@ func (s *Source) chunkJSONEnumerator( if err != nil { return fmt.Errorf("unable to open file: %w", err) } - defer enumeratorFile.Close() + defer func() { _ = enumeratorFile.Close() }() + + if strings.HasSuffix(path, ".zst") || strings.HasSuffix(path, ".zstd") { + r, err := zstd.NewReader(enumeratorFile) + if err != nil { + return fmt.Errorf("unable to open zstd reader: %w", err) + } + defer r.Close() + return s.chunkJSONEnumeratorReader(ctx, r, reporter) + } else { + return s.chunkJSONEnumeratorReader(ctx, enumeratorFile, reporter) + } - return s.chunkJSONEnumeratorReader(ctx, enumeratorFile, chunksChan) } diff --git a/pkg/sources/json_enumerator/json_enumerator_test.go b/pkg/sources/json_enumerator/json_enumerator_test.go index 91f777a62cb3..11d8f6ad91a6 100644 --- a/pkg/sources/json_enumerator/json_enumerator_test.go +++ b/pkg/sources/json_enumerator/json_enumerator_test.go @@ -1,6 +1,8 @@ package json_enumerator import ( + "archive/zip" + "bytes" "encoding/json" "io" "strings" @@ -12,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) @@ -102,7 +105,7 @@ func TestScanEnumerator(t *testing.T) { defer close(chunksChan) ctx := context.WithLogger(t.Context(), logr.Discard()) source := &Source{} - workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, chunksChan) + workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, sources.ChanReporter{Ch: chunksChan}) }() enc := json.NewEncoder(writeJSON) @@ -127,3 +130,87 @@ func TestScanEnumerator(t *testing.T) { }) } } + +// makeZip builds an in-memory zip from the given entries. A large stored padding +// entry is written first so mimetype classifies the bytes as a generic zip, +// leaving APK identification to the content-based deep scan. +func makeZip(t *testing.T, files map[string]string) []byte { + t.Helper() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + padding, err := zw.CreateHeader(&zip.FileHeader{Name: "padding.txt", Method: zip.Store}) + require.NoError(t, err) + _, err = padding.Write(bytes.Repeat([]byte("A"), 5000)) + require.NoError(t, err) + + for name, content := range files { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte(content)) + require.NoError(t, err) + } + + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// TestScanEnumeratorAPKContentRouting proves json-enumerator input is routed by +// file content, not by any filename/extension. The records carry no filename. +// A zip whose contents mark it as an APK (AndroidManifest.xml + classes.dex) is +// routed to the APK handler; because this synthetic archive lacks resources.arsc +// the APK handler yields no chunks, so the plaintext secret is not surfaced. +// The identical secret in a plain zip (no APK markers) is handled generically +// and surfaces the secret. +func TestScanEnumeratorAPKContentRouting(t *testing.T) { + feature.EnableAPKHandler.Store(true) + t.Cleanup(func() { feature.EnableAPKHandler.Store(false) }) + + secret := secretPart1 + secretPart2 + + apkZip := makeZip(t, map[string]string{ + "AndroidManifest.xml": "placeholder", + "classes.dex": "placeholder", + "assets/secret.txt": secret, + }) + plainZip := makeZip(t, map[string]string{ + "assets/secret.txt": secret, + }) + + run := func(data []byte) (string, error) { + readJSON, writeJSON := io.Pipe() + chunksChan := make(chan *sources.Chunk, 16) + + var workerError error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + defer close(chunksChan) + ctx := context.WithLogger(t.Context(), logr.Discard()) + source := &Source{} + workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, sources.ChanReporter{Ch: chunksChan}) + }() + + // No filename is set on the record; routing is content-based only. + enc := json.NewEncoder(writeJSON) + require.NoError(t, enc.Encode(&jsonEntry{Metadata: makeRawMessage(t, "{}"), Data: data})) + require.NoError(t, writeJSON.Close()) + + found := "" + for chunk := range chunksChan { + found += string(chunk.Data) + } + wg.Wait() + return found, workerError + } + + foundAPK, err := run(apkZip) + require.NoError(t, err) + assert.NotContains(t, foundAPK, secret, "APK-content archive should be routed to the APK handler") + + foundZip, err := run(plainZip) + require.NoError(t, err) + assert.Contains(t, foundZip, secret, "non-APK zip should be handled generically and surface the secret") +} diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index b6e6140afe7c..c0a7fccc19ae 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -430,7 +430,7 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c metadata.FolderID = item.Uid // check if there are any requests in the folder if item.Request.Method != "" { - metadata.FolderName = strings.Replace(metadata.FolderName, (" > " + item.Name), "", -1) + metadata.FolderName = strings.ReplaceAll(metadata.FolderName, (" > " + item.Name), "") metadata.FolderID = parentItemId if metadata.FolderID == "" { metadata.FolderName = "" @@ -470,7 +470,7 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, // Prep direct links. Ignore updating link if it's a local JSON file if !metadata.fromLocal { - metadata.Link = LINK_BASE_URL + (strings.Replace(metadata.Type, " > event", "", -1)) + "/" + metadata.FullID + metadata.Link = LINK_BASE_URL + (strings.ReplaceAll(metadata.Type, " > event", "")) + "/" + metadata.FullID if event.Listen == "prerequest" { metadata.Link += "?tab=pre-request-scripts" } else { @@ -817,13 +817,13 @@ func unpackWorkspace(workspacePath string) (Workspace, error) { if err != nil { return workspace, err } - defer r.Close() + defer func() { _ = r.Close() }() for _, file := range r.File { rc, err := file.Open() if err != nil { return workspace, err } - defer rc.Close() + defer func() { _ = rc.Close() }() contents, err := io.ReadAll(rc) if err != nil { return workspace, err diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 6ecb9f44067f..8379e8c56891 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -283,7 +283,7 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st } return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() c.Metrics.apiRequests.WithLabelValues(urlString).Inc() diff --git a/pkg/sources/postman/substitution.go b/pkg/sources/postman/substitution.go index 9f68f4f6efd9..491ac119c121 100644 --- a/pkg/sources/postman/substitution.go +++ b/pkg/sources/postman/substitution.go @@ -2,7 +2,7 @@ package postman import ( "fmt" - "regexp" + regexp "github.com/wasilibs/go-re2" "strings" "github.com/trufflesecurity/trufflehog/v3/pkg/context" diff --git a/pkg/sources/s3/README.md b/pkg/sources/s3/README.md new file mode 100644 index 000000000000..bf0e6531a25c --- /dev/null +++ b/pkg/sources/s3/README.md @@ -0,0 +1,92 @@ +# S3 source + +Scans objects in S3 buckets, or in any S3-compatible service. + +## Scan paths + +The source implements both the plain `Source` interface and `SourceUnitEnumChunker`, where one unit is one bucket (plus the role used to reach it). + +- `Chunks` walks every bucket in sequence. +- `Enumerate` reports each bucket as a unit, and `ChunkUnit` scans one bucket. + +Both end up in `scanBucket` -> `pageChunker`, so per-object behavior is identical either way. The CLI always takes the unit path, because `main.go` enables source units and the source supports them. + +## Credentials + +Set by which flags are present: + +| Flags | Credentials used | +| --- | --- | +| none | anonymous, for public buckets only | +| `--cloud-environment` | the AWS SDK's normal chain: env vars, `~/.aws/credentials`, instance metadata | +| `--key` and `--secret` | those static credentials | +| `--key`, `--secret`, `--session-token` | those temporary credentials | + +The default is anonymous, not your local AWS profile. Scanning a private bucket without `--cloud-environment` or explicit keys gives `403 AccessDenied`. + +`--role-arn` assumes a role before scanning, and can be repeated. With roles and no `--bucket`, the source tries every bucket each role can list, so `AccessDenied` on buckets a role cannot reach is expected and does not fail the scan. + +## Choosing buckets + +`--bucket` scans only the named buckets. `--ignore-bucket` scans everything else. They cannot be combined; doing so fails at startup. Bucket names are matched exactly, not as globs or prefixes. + +## Choosing objects within a bucket + +Four flags, all repeatable, all matched against the object key before the object is downloaded. A skipped object costs no `GetObject` request. + +- `--include-prefix` / `--exclude-prefix` match a literal key prefix. They can be used together to scan a subtree while leaving part of it out, and an exclude match always wins. +- `--include-extension` / `--exclude-extension` match the file extension, written without a leading dot. They cannot be used together, since listing what to scan already excludes everything else. Matching is case insensitive. + +Prefixes are literal, so `--include-prefix=log` matches `logs/app.txt` and `logs-archive/app.txt` alike. + +The two kinds are combined with AND: an object has to pass both to be scanned. Entries are trimmed, and blank ones are ignored. + +Include prefixes are also sent to S3 as `ListObjectsV2Input.Prefix`. A bucket is listed once per include prefix instead of in full, so the listing covers only the keys they select, and each prefix costs at least one `ListObjectsV2` request per bucket even when nothing is under it. A prefix under another, like `src/vendor/` under `src/`, is dropped, since the shorter one already lists it. Directory buckets are listed in full, because they only accept a prefix that ends in `/`. + +Exclude prefixes and extensions have no server-side form, so they are matched after listing. They save `GetObject` requests but not `ListObjectsV2` requests. + +Three edge cases worth knowing: + +- Only the last extension counts. `path.Ext("backup.tar.gz")` is `.gz`, so `--exclude-extension=tar.gz` never matches, while `--exclude-extension=gz` does. +- A key with no extension, like `Makefile`, matches no extension entry, so `--include-extension` skips it and `--exclude-extension` keeps it. +- A dotfile is treated as all extension. `.env` has extension `env`, so `--exclude-extension=env` skips it, and any `--include-extension` list that omits `env` skips it too. That is easy to do by accident, and `.env` files are a common source of secrets. + +Filters apply to object keys only, not to files inside an archive. `--exclude-extension=mp4` skips `clip.mp4` in the bucket but not a `clip.mp4` packed inside `media.zip`, since archive contents are filtered by the shared deny list instead. Excluding `zip` avoids expanding the archive at all. + +Prefixes are matched with `strings.HasPrefix`, so an exclude prefix naming one key also excludes anything that key is a prefix of: `--exclude-prefix=foo/bar.txt` excludes `foo/bar.txt.bak` as well. There is no exact-key match. + +The same prefixes and extensions apply to every bucket in a scan. There is no way to scope a prefix to one bucket. + +If a filter excludes every object in a bucket, the scan logs `Scanned no objects in bucket` with the count. If nothing is listed under the include prefixes at all, it logs `Found no objects under the include prefixes`. Either way, a mistyped prefix does not look like a clean scan of an empty bucket. + +## Objects skipped regardless of configuration + +- Glacier and Glacier Instant Retrieval storage classes +- Larger than `--max-object-size`, which defaults to 250 MiB and cannot be raised above it +- Empty +- Extensions in the shared deny list in `pkg/common/vars.go`, which already covers images, audio, and video +- Keys ending in `/`, which are directory markers + +## Custom endpoints + +`--endpoint` points the scan at an S3-compatible service instead of AWS S3. Setting it changes three things: + +- The client uses path-style addressing, since these services rarely publish the wildcard DNS that virtual-hosted addressing needs. +- Per-bucket region discovery is skipped. `GetBucketRegion` is AWS-only, and a custom endpoint serves its own buckets, so requests sign with `--region` instead. +- Object links in finding metadata point at the endpoint rather than `amazonaws.com`, so they resolve the way the scan read them. + +The endpoint is parsed in `Init`, so a malformed one fails at startup rather than as an SDK error mid-scan. A missing scheme is assumed to be `https`. + +`--region` sets the region used to sign requests and defaults to `us-east-1`. On AWS each bucket's real region is discovered automatically, so it mostly matters with a custom endpoint. + +## Resumption + +`Checkpointer` tracks which objects in the current page of up to 1000 have finished, and records the highest consecutively completed key as `StartAfter` in `Progress.EncodedResumeInfo`, along with the bucket and role. An interrupted scan resumes from there. + +Only consecutive completions are checkpointed. If objects 0 to 5 and 7 to 8 are done but 6 is not, the checkpoint stops at 5, so resuming may rescan a few objects but never misses one. + +With include prefixes the scan lists each in turn. They are listed in key order and none is under another, so keys still arrive in ascending order and the checkpoint stays a single key. Resuming passes it as `StartAfter` to every prefix, and S3 returns nothing for the prefixes already finished. A checkpoint from a scan that listed the whole bucket resumes the same way. + +This matters when adding a new skip to `pageChunker`: a skipped object still has to call `UpdateObjectCompletion`. Skipping without it stalls the checkpoint at the first skipped object, and a resumed scan then redoes everything after it. + +The OSS CLI does not persist progress between runs, so resumption only takes effect where something stores and returns `EncodedResumeInfo`. diff --git a/pkg/sources/s3/filter.go b/pkg/sources/s3/filter.go new file mode 100644 index 000000000000..cd56d2406eab --- /dev/null +++ b/pkg/sources/s3/filter.go @@ -0,0 +1,157 @@ +package s3 + +import ( + "errors" + "path" + "slices" + "strings" +) + +// objectFilter narrows a scan to the configured key prefixes and file extensions. +// An object must pass both axes to be scanned. +// +// The two axes treat a populated include and exclude list differently. Prefixes +// accept both at once, so a scan can cover a subtree minus one directory, and +// exclusion wins on a key that matches both. Extensions accept only one, because +// naming the extensions to scan already excludes every other one; both together is +// rejected in Init. Buckets reject both lists too, so prefixes are the exception. +// +// The zero value and a nil *objectFilter both include everything. +type objectFilter struct { + // Sorted, and none starts with another, so a scan can list each in turn and + // still see keys in order. + includePrefixes []string + excludePrefixes []string + + // Lowercase extensions without a leading dot, sorted and deduplicated. At most + // one is populated. + includeExtensions []string + excludeExtensions []string +} + +// newObjectFilter builds a filter from the configured lists. Extensions may be +// written with or without a leading dot and in any case. Entries are trimmed and +// blank ones dropped, since a padded entry would match nothing and an empty prefix +// would match every key. +func newObjectFilter(includePrefixes, excludePrefixes, includeExtensions, excludeExtensions []string) (*objectFilter, error) { + // Compare the cleaned lists rather than the raw ones, so that a list holding + // nothing but blanks does not count as populated. + include, exclude := cleanExtensions(includeExtensions), cleanExtensions(excludeExtensions) + if len(include) > 0 && len(exclude) > 0 { + return nil, errors.New("either an extension include list or an extension exclude list can be specified, but not both") + } + + return &objectFilter{ + includePrefixes: outermostPrefixes(cleanEntries(includePrefixes, strings.TrimSpace)), + excludePrefixes: cleanEntries(excludePrefixes, strings.TrimSpace), + includeExtensions: include, + excludeExtensions: exclude, + }, nil +} + +// shouldInclude reports whether the object key passes both filter axes. +func (f *objectFilter) shouldInclude(key string) bool { + if f == nil { + return true + } + return f.passesPrefixes(key) && f.passesExtensions(key) +} + +// listPrefixes returns the include prefixes for S3 to list in place of the whole +// bucket, or nil when the whole bucket is listed. +func (f *objectFilter) listPrefixes() []string { + if f == nil { + return nil + } + return f.includePrefixes +} + +func (f *objectFilter) passesPrefixes(key string) bool { + for _, prefix := range f.excludePrefixes { + if strings.HasPrefix(key, prefix) { + return false + } + } + if len(f.includePrefixes) == 0 { + return true + } + for _, prefix := range f.includePrefixes { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +// passesExtensions applies the extension axis. A key with no extension matches no +// configured entry, so it fails a populated include list and passes an exclude list. +func (f *objectFilter) passesExtensions(key string) bool { + if len(f.includeExtensions) == 0 && len(f.excludeExtensions) == 0 { + return true + } + + // path.Ext rather than filepath.Ext: S3 keys always separate on "/", whatever + // the host OS uses. + ext := normalizeExtension(path.Ext(key)) + if len(f.excludeExtensions) > 0 { + return !slices.Contains(f.excludeExtensions, ext) + } + return slices.Contains(f.includeExtensions, ext) +} + +// isConfigured reports whether any filtering is in effect. +func (f *objectFilter) isConfigured() bool { + if f == nil { + return false + } + return len(f.includePrefixes) > 0 || len(f.excludePrefixes) > 0 || + len(f.includeExtensions) > 0 || len(f.excludeExtensions) > 0 +} + +// normalizeExtension trims either side of the dot, so that a padded configured +// value cannot silently match nothing. It runs on configured values and on the +// extension read from a key alike, keeping both sides of a comparison normalized +// the same way. +func normalizeExtension(ext string) string { + return strings.ToLower(strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(ext), "."))) +} + +// cleanExtensions normalizes configured extensions and sorts them, so that logging +// the effective filter is stable rather than following the order they were given. +func cleanExtensions(values []string) []string { + exts := cleanEntries(values, normalizeExtension) + slices.Sort(exts) + return slices.Compact(exts) +} + +// outermostPrefixes sorts prefixes and drops each one that starts with another, +// since the shorter one already covers every key under it. Once sorted, a prefix +// that starts with any kept prefix starts with the last one kept. +func outermostPrefixes(prefixes []string) []string { + slices.Sort(prefixes) + var kept []string + for _, prefix := range prefixes { + if len(kept) == 0 || !strings.HasPrefix(prefix, kept[len(kept)-1]) { + kept = append(kept, prefix) + } + } + return kept +} + +// cleanEntries normalizes each entry and drops the ones that come out empty, +// returning nil when nothing is left. +func cleanEntries(values []string, normalize func(string) string) []string { + if len(values) == 0 { + return nil + } + kept := make([]string, 0, len(values)) + for _, value := range values { + if normalized := normalize(value); normalized != "" { + kept = append(kept, normalized) + } + } + if len(kept) == 0 { + return nil + } + return kept +} diff --git a/pkg/sources/s3/filter_test.go b/pkg/sources/s3/filter_test.go new file mode 100644 index 000000000000..5ccaaeee86ad --- /dev/null +++ b/pkg/sources/s3/filter_test.go @@ -0,0 +1,129 @@ +package s3 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestObjectFilter(t *testing.T, includePrefixes, excludePrefixes, includeExts, excludeExts []string) *objectFilter { + t.Helper() + + filter, err := newObjectFilter(includePrefixes, excludePrefixes, includeExts, excludeExts) + require.NoError(t, err) + return filter +} + +func TestObjectFilter_Unconfigured(t *testing.T) { + filter := newTestObjectFilter(t, nil, nil, nil, nil) + + assert.True(t, filter.shouldInclude("any/key.zip")) + assert.True(t, filter.shouldInclude("Makefile")) + assert.True(t, (*objectFilter)(nil).shouldInclude("any/key.zip")) + assert.Nil(t, (*objectFilter)(nil).listPrefixes()) +} + +func TestObjectFilter_ExcludePrefix(t *testing.T) { + filter := newTestObjectFilter(t, nil, []string{"projects/abc/"}, nil, nil) + + assert.False(t, filter.shouldInclude("projects/abc/config.yaml")) + assert.True(t, filter.shouldInclude("projects/xyz/config.yaml")) +} + +func TestObjectFilter_IncludePrefix(t *testing.T) { + filter := newTestObjectFilter(t, []string{"infra/"}, nil, nil, nil) + + assert.True(t, filter.shouldInclude("infra/main.tf")) + assert.False(t, filter.shouldInclude("media/clip.mp4")) + + // A prefix is matched literally, so it is not confined to a path boundary. + assert.True(t, newTestObjectFilter(t, []string{"log"}, nil, nil, nil).shouldInclude("logs-archive/app.txt")) +} + +func TestObjectFilter_IncludeAndExcludePrefixes(t *testing.T) { + filter := newTestObjectFilter(t, []string{"src/"}, []string{"src/vendor/"}, nil, nil) + + assert.True(t, filter.shouldInclude("src/main.go")) + assert.False(t, filter.shouldInclude("src/vendor/dep.go"), "exclusion takes precedence") + assert.False(t, filter.shouldInclude("docs/readme.md"), "matches no include prefix") +} + +// A scan lists the include prefixes in turn, so they must come out sorted with +// none under another, or keys would be listed twice or out of order. +func TestObjectFilter_IncludePrefixesAreOutermost(t *testing.T) { + include := []string{"src/vendor/", "src/", "docs/", "src/", "a", "ab", "ac"} + filter := newTestObjectFilter(t, include, nil, nil, nil) + + assert.Equal(t, []string{"a", "docs/", "src/"}, filter.listPrefixes()) + assert.True(t, filter.shouldInclude("src/vendor/dep.go"), "a dropped prefix is still covered") + assert.True(t, filter.shouldInclude("ac/key.txt")) +} + +func TestObjectFilter_ExcludeExtension(t *testing.T) { + filter := newTestObjectFilter(t, nil, nil, nil, []string{"zip", ".MP4"}) + + assert.False(t, filter.shouldInclude("build/artifact.zip")) + assert.False(t, filter.shouldInclude("build/ARTIFACT.ZIP"), "matching is case insensitive") + assert.False(t, filter.shouldInclude("media/clip.mp4"), "a configured leading dot is tolerated") + assert.True(t, filter.shouldInclude("infra/main.tf")) + assert.True(t, filter.shouldInclude("Makefile"), "a key with no extension is not excluded") +} + +func TestObjectFilter_IncludeExtension(t *testing.T) { + filter := newTestObjectFilter(t, nil, nil, []string{"tf", "yaml"}, nil) + + assert.True(t, filter.shouldInclude("infra/main.tf")) + assert.True(t, filter.shouldInclude("infra/vars.yaml")) + assert.False(t, filter.shouldInclude("infra/state.json")) + assert.False(t, filter.shouldInclude("Makefile"), "a key with no extension matches no entry") +} + +func TestObjectFilter_BothExtensionListsRejected(t *testing.T) { + _, err := newObjectFilter(nil, nil, []string{"tf"}, []string{"zip"}) + + assert.Error(t, err) +} + +func TestObjectFilter_AxesCombine(t *testing.T) { + filter := newTestObjectFilter(t, []string{"src/"}, nil, nil, []string{"zip"}) + + assert.True(t, filter.shouldInclude("src/main.go")) + assert.False(t, filter.shouldInclude("src/bundle.zip"), "passes the prefix axis, fails the extension axis") + assert.False(t, filter.shouldInclude("docs/guide.md"), "passes the extension axis, fails the prefix axis") +} + +// Enterprise passes these lists as newline separated text, which can yield blank +// entries. A blank exclude prefix would otherwise match every key and skip the +// whole bucket. +func TestObjectFilter_BlankEntriesIgnored(t *testing.T) { + filter := newTestObjectFilter(t, nil, []string{""}, nil, []string{""}) + + assert.True(t, filter.shouldInclude("any/key.zip")) +} + +// A list holding only blanks is not a populated list, so it cannot conflict with +// the opposite list. +func TestObjectFilter_BlankExtensionListIsNotAConflict(t *testing.T) { + filter := newTestObjectFilter(t, nil, nil, []string{""}, []string{"zip"}) + + assert.False(t, filter.shouldInclude("build/artifact.zip")) + assert.True(t, filter.shouldInclude("infra/main.tf")) +} + +// Padded entries reach the filter from config files and shell arguments. Without +// trimming they match nothing, so an exclusion the user configured would silently +// do nothing. +func TestObjectFilter_EntriesAreTrimmed(t *testing.T) { + prefixFilter := newTestObjectFilter(t, nil, []string{"archive/\r"}, nil, nil) + assert.False(t, prefixFilter.shouldInclude("archive/a.txt")) + + extFilter := newTestObjectFilter(t, nil, nil, nil, []string{" zip "}) + assert.False(t, extFilter.shouldInclude("build/artifact.zip")) + + dotPadded := newTestObjectFilter(t, nil, nil, nil, []string{". zip"}) + assert.False(t, dotPadded.shouldInclude("build/artifact.zip")) + + whitespaceOnly := newTestObjectFilter(t, nil, []string{" "}, nil, nil) + assert.True(t, whitespaceOnly.shouldInclude("any/key.txt")) +} diff --git a/pkg/sources/s3/metrics.go b/pkg/sources/s3/metrics.go index 534241477266..70185032b49a 100644 --- a/pkg/sources/s3/metrics.go +++ b/pkg/sources/s3/metrics.go @@ -15,6 +15,10 @@ type metricsCollector interface { RecordObjectSkipped(bucket, reason string, sizeBytes float64) RecordObjectError(bucket string) + // Bucket metrics. + + RecordBucketListError(bucket, roleArn string) + // Role metrics. RecordRoleScanned(roleArn string) @@ -22,11 +26,12 @@ type metricsCollector interface { } type collector struct { - objectsScanned *prometheus.HistogramVec - objectsSkipped *prometheus.HistogramVec - objectsErrors *prometheus.CounterVec - rolesScanned *prometheus.GaugeVec - bucketsPerRole *prometheus.GaugeVec + objectsScanned *prometheus.HistogramVec + objectsSkipped *prometheus.HistogramVec + objectsErrors *prometheus.CounterVec + bucketListErrors *prometheus.CounterVec + rolesScanned *prometheus.GaugeVec + bucketsPerRole *prometheus.GaugeVec } var metricsInstance metricsCollector @@ -58,6 +63,13 @@ func init() { Help: "Total number of errors encountered during S3 scan", }, []string{"bucket"}), + bucketListErrors: promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: common.MetricsNamespace, + Subsystem: common.MetricsSubsystem, + Name: "bucket_list_errors_total", + Help: "Total number of failures to list objects in an S3 bucket, by bucket and role", + }, []string{"bucket", "role_arn"}), + rolesScanned: promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: common.MetricsNamespace, Subsystem: common.MetricsSubsystem, @@ -86,6 +98,13 @@ func (c *collector) RecordObjectError(bucket string) { c.objectsErrors.WithLabelValues(bucket).Inc() } +func (c *collector) RecordBucketListError(bucket, roleArn string) { + if roleArn == "" { + roleArn = defaultRoleARN + } + c.bucketListErrors.WithLabelValues(bucket, roleArn).Inc() +} + const defaultRoleARN = "default" func (c *collector) RecordRoleScanned(roleArn string) { diff --git a/pkg/sources/s3/paginator.go b/pkg/sources/s3/paginator.go new file mode 100644 index 000000000000..05870e4d7748 --- /dev/null +++ b/pkg/sources/s3/paginator.go @@ -0,0 +1,71 @@ +package s3 + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" +) + +// directoryBucketSuffix ends every S3 Express directory bucket name, and AWS reserves it for them. +const directoryBucketSuffix = "--x-s3" + +// listInputs returns the listings that cover what a scan of the bucket can include: one per include +// prefix, so S3 skips the keys outside them, or one of the whole bucket. +// +// The include prefixes are sorted and none starts with another, so listing them in turn returns keys +// in ascending order, just as one listing of the bucket would. That keeps the checkpoint a single +// start-after key, and passing it to every listing resumes exactly where the scan stopped: S3 +// returns nothing for the prefixes the scan already finished. +func (s *Source) listInputs(bucket string, startAfter *string) []*s3.ListObjectsV2Input { + prefixes := s.objectFilter.listPrefixes() + // Directory buckets only accept a prefix that ends in "/", so they are listed whole and filtered after. + if len(prefixes) == 0 || strings.HasSuffix(bucket, directoryBucketSuffix) { + return []*s3.ListObjectsV2Input{{Bucket: &bucket, StartAfter: startAfter}} + } + + inputs := make([]*s3.ListObjectsV2Input, len(prefixes)) + for i, prefix := range prefixes { + inputs[i] = &s3.ListObjectsV2Input{Bucket: &bucket, Prefix: aws.String(prefix), StartAfter: startAfter} + } + return inputs +} + +// bucketPaginator pages through several listings in turn, as if they were one. +type bucketPaginator struct { + client *s3.Client + // inputs[0] is the listing in progress, and the rest are still to come. + inputs []*s3.ListObjectsV2Input + pages *s3.ListObjectsV2Paginator +} + +// newBucketPaginator starts with the first of inputs, which must not be empty. +func newBucketPaginator(client *s3.Client, inputs []*s3.ListObjectsV2Input) *bucketPaginator { + return &bucketPaginator{client: client, inputs: inputs, pages: s3.NewListObjectsV2Paginator(client, inputs[0])} +} + +// HasMorePages reports whether any listing has pages left, moving on to the next listing when the +// current one is done. +func (p *bucketPaginator) HasMorePages() bool { + for !p.pages.HasMorePages() { + if len(p.inputs) == 1 { + return false + } + p.inputs = p.inputs[1:] + p.pages = s3.NewListObjectsV2Paginator(p.client, p.inputs[0]) + } + return true +} + +// NextPage returns the next page of the current listing. +func (p *bucketPaginator) NextPage(ctx context.Context) (*s3.ListObjectsV2Output, error) { + page, err := p.pages.NextPage(ctx) + if err != nil && p.inputs[0].Prefix != nil { + // A policy can allow listing some prefixes and not others, so say which one failed. + err = fmt.Errorf("could not list prefix %q: %w", *p.inputs[0].Prefix, err) + } + return page, err +} diff --git a/pkg/sources/s3/paginator_test.go b/pkg/sources/s3/paginator_test.go new file mode 100644 index 000000000000..b305802d483b --- /dev/null +++ b/pkg/sources/s3/paginator_test.go @@ -0,0 +1,249 @@ +package s3 + +import ( + "slices" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/go-logr/logr/funcr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + "github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest" +) + +func TestSource_ListInputs(t *testing.T) { + startAfter := "infra/main.tf" + + tests := []struct { + name string + bucket string + prefixes []string + wantPrefixes []string + }{ + {name: "no include prefixes", bucket: "bucket", wantPrefixes: []string{""}}, + { + name: "one listing per include prefix, in key order", + bucket: "bucket", + prefixes: []string{"src/", "infra/"}, + wantPrefixes: []string{"infra/", "src/"}, + }, + { + name: "directory bucket", + bucket: "data--usw2-az1--x-s3", + prefixes: []string{"src"}, + wantPrefixes: []string{""}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Source{objectFilter: newTestObjectFilter(t, tt.prefixes, nil, nil, nil)} + + var gotPrefixes []string + for _, input := range s.listInputs(tt.bucket, &startAfter) { + assert.Equal(t, tt.bucket, aws.ToString(input.Bucket)) + assert.Equal(t, startAfter, aws.ToString(input.StartAfter), "every listing resumes at the checkpoint") + gotPrefixes = append(gotPrefixes, aws.ToString(input.Prefix)) + } + assert.Equal(t, tt.wantPrefixes, gotPrefixes) + }) + } +} + +// prefixTestObjects has keys under the include prefixes the tests below use, one of them nested in +// another, and keys between and around them that no include prefix covers. +func prefixTestObjects() map[string]string { + objects := make(map[string]string) + for _, key := range []string{ + "configs/app.yaml", + "infra/main.tf", "infra/vars.tf", + "logs/1.log", "logs/2.log", "logs/3.log", "logs/4.log", + "src/main.go", "src/vendor/dep.go", + "tests/main_test.go", + } { + objects[key] = "contents of " + key + } + return objects +} + +func withIncludePrefixes(prefixes ...string) func(*sourcespb.S3) { + return func(conn *sourcespb.S3) { conn.IncludePrefixes = prefixes } +} + +// scannedKeys counts the chunks reported for each object. +func scannedKeys(chunks []sources.Chunk) map[string]int { + keys := make(map[string]int) + for _, chunk := range chunks { + keys[chunk.SourceMetadata.GetS3().GetFile()]++ + } + return keys +} + +func TestSource_ChunkUnit_ListsOnlyIncludePrefixes(t *testing.T) { + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects()}}} + s := newFakeS3Source(t, fake, withIncludePrefixes("src/", "infra/", "src/vendor/")) + + reporter := sourcestest.TestReporter{} + require.NoError(t, s.ChunkUnit(context.Background(), S3SourceUnit{Bucket: "bucket"}, &reporter)) + + want := map[string]int{"infra/main.tf": 1, "infra/vars.tf": 1, "src/main.go": 1, "src/vendor/dep.go": 1} + assert.Equal(t, want, scannedKeys(reporter.Chunks), "each object under a prefix is scanned once") + // infra/ and src/ fit a page each, where listing the whole bucket takes six. + assert.EqualValues(t, 2, fake.listCalls.Load()) +} + +// A checkpoint is the last key the scan finished, whether it lies under an include prefix or not, as +// one left by a scan that listed the whole bucket may. +func TestSource_ChunkUnit_ResumesAcrossIncludePrefixes(t *testing.T) { + tests := []struct { + name string + startAfter string + want []string + }{ + { + name: "before every prefix", + startAfter: "a", + want: []string{"infra/main.tf", "infra/vars.tf", "src/main.go", "src/vendor/dep.go"}, + }, + { + name: "within a prefix", + startAfter: "infra/main.tf", + want: []string{"infra/vars.tf", "src/main.go", "src/vendor/dep.go"}, + }, + {name: "between prefixes", startAfter: "logs/3.log", want: []string{"src/main.go", "src/vendor/dep.go"}}, + {name: "before a nested prefix", startAfter: "src/main.go", want: []string{"src/vendor/dep.go"}}, + {name: "after every prefix", startAfter: "tests/main_test.go"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects()}}} + s := newFakeS3Source(t, fake, withIncludePrefixes("infra/", "src/", "src/vendor/")) + + unit := S3SourceUnit{Bucket: "bucket"} + unitID, _ := unit.SourceUnitID() + s.SetEncodedResumeInfoFor(unitID, tt.startAfter) + + reporter := sourcestest.TestReporter{} + require.NoError(t, s.ChunkUnit(context.Background(), unit, &reporter)) + + want := make(map[string]int) + for _, key := range tt.want { + want[key] = 1 + } + assert.Equal(t, want, scannedKeys(reporter.Chunks)) + }) + } +} + +// S3 applies the include prefixes itself, so a mistyped one leaves no object for the filter to +// exclude. The scan must still say it found nothing, or it looks like a clean scan of an empty bucket, +// but only when the prefixes truly listed nothing. +func TestSource_ChunkUnit_ReportsIncludePrefixesListingNothing(t *testing.T) { + tests := []struct { + name string + prefix string + startAfter string + want bool + }{ + {name: "mistyped prefix", prefix: "infa/", want: true}, + {name: "resumed after every prefix", prefix: "infra/", startAfter: "infra/vars.tf"}, + {name: "listed objects the scan skips", prefix: "empty"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := prefixTestObjects() + objects["empty.txt"] = "" + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: objects}}} + s := newFakeS3Source(t, fake, withIncludePrefixes(tt.prefix)) + + unit := S3SourceUnit{Bucket: "bucket"} + unitID, _ := unit.SourceUnitID() + s.SetEncodedResumeInfoFor(unitID, tt.startAfter) + + var mu sync.Mutex + var logged []string + ctx := context.WithLogger(context.Background(), funcr.New(func(_, args string) { + mu.Lock() + defer mu.Unlock() + logged = append(logged, args) + }, funcr.Options{})) + + require.NoError(t, s.ChunkUnit(ctx, unit, &sourcestest.TestReporter{})) + + assert.Equal(t, tt.want, slices.ContainsFunc(logged, func(line string) bool { + return strings.Contains(line, "Found no objects under the include prefixes") + })) + }) + } +} + +// A policy can allow listing some prefixes and not others, so a failed listing must say which. +func TestBucketPaginator_ErrorNamesPrefix(t *testing.T) { + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects(), listPrefix: "infra/"}}} + s := newFakeS3Source(t, fake, withIncludePrefixes("infra/", "src/")) + client, err := s.newClient(context.Background(), s.defaultRegion(), "") + require.NoError(t, err) + + paginator := newBucketPaginator(client, s.listInputs("bucket", nil)) + var listErr error + for paginator.HasMorePages() && listErr == nil { + _, listErr = paginator.NextPage(context.Background()) + } + + assert.ErrorContains(t, listErr, `could not list prefix "src/"`) +} + +func TestSource_ChunkUnit_CountsOnlyIncludePrefixes(t *testing.T) { + enableUnitProgress(t) + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects()}}} + s := newFakeS3Source(t, fake, withIncludePrefixes("infra/", "src/")) + + unit := S3SourceUnit{Bucket: "bucket"} + require.NoError(t, s.ChunkUnit(context.Background(), unit, &sourcestest.TestReporter{})) + + unitID, _ := unit.SourceUnitID() + got, _ := s.GetUnitProgressFor(unitID) + assert.Equal(t, sources.UnitProgressReady, got.State) + assert.EqualValues(t, 4, got.ItemsTotal) + assert.EqualValues(t, 4, fake.listCalls.Load(), "the count lists the same two prefixes as the scan") +} + +// A policy can limit listing to some prefixes. Validate must list the way the scan does, or it +// rejects access the scan has. +func TestSource_Validate_ListsIncludePrefix(t *testing.T) { + tests := []struct { + name string + prefixes []string + wantErr bool + }{ + {name: "whole bucket", wantErr: true}, + {name: "include prefix", prefixes: []string{"infra/"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := &fakeS3{buckets: map[string]fakeBucket{ + "bucket": {objects: prefixTestObjects(), listPrefix: "infra/"}, + }} + s := newFakeS3Source(t, fake, withIncludePrefixes(tt.prefixes...), func(conn *sourcespb.S3) { + conn.Buckets = []string{"bucket"} + }) + + errs := s.Validate(context.Background()) + + if tt.wantErr { + assert.NotEmpty(t, errs) + } else { + assert.Empty(t, errs) + } + }) + } +} diff --git a/pkg/sources/s3/progress.go b/pkg/sources/s3/progress.go new file mode 100644 index 000000000000..aa2d11a322f7 --- /dev/null +++ b/pkg/sources/s3/progress.go @@ -0,0 +1,152 @@ +package s3 + +import ( + "sync" + + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +// unitProgress tracks how far a unit scan has got through its bucket and publishes it on the source's +// Progress under the unit's ID. Each ChunkUnit call has its own, so buckets scanned at the same time +// never mix their counts. +type unitProgress struct { + unitID string + progress *sources.Progress + + mu sync.Mutex + current sources.UnitProgress + finished bool +} + +func newUnitProgress(unitID string, progress *sources.Progress) *unitProgress { + return &unitProgress{unitID: unitID, progress: progress} +} + +// update applies f and publishes the result. Publishing under the lock keeps an older value from +// replacing a newer one. Nothing changes once the unit has finished. +func (p *unitProgress) update(f func(current *sources.UnitProgress)) { + p.mu.Lock() + defer p.mu.Unlock() + if p.finished { + return + } + f(&p.current) + p.progress.SetUnitProgressFor(p.unitID, p.current) +} + +// objectDone records that the scan is finished with an object it downloads, whether or not the +// download worked. It does nothing on a nil receiver, which is how a legacy scan runs. +func (p *unitProgress) objectDone(size int64) { + if p == nil { + return + } + p.update(func(current *sources.UnitProgress) { + current.ItemsDone++ + current.BytesDone += uint64(max(size, 0)) + }) +} + +// counted adds a listed page to the totals. Objects an earlier run finished count as done too. +func (p *unitProgress) counted(items, bytes, resumedItems, resumedBytes uint64) { + p.update(func(current *sources.UnitProgress) { + current.ItemsTotal += items + current.BytesTotal += bytes + current.ItemsDone += resumedItems + current.BytesDone += resumedBytes + current.ResumedBytes += resumedBytes + }) +} + +func (p *unitProgress) setState(state sources.UnitProgressState) { + p.update(func(current *sources.UnitProgress) { current.State = state }) +} + +// finish settles the totals once the scan of the bucket has ended. scannedWholeBucket means the scan +// listed the bucket from the start to the end, so it has seen every object there is. +func (p *unitProgress) finish(scannedWholeBucket bool) { + p.update(func(current *sources.UnitProgress) { + switch { + case current.State == sources.UnitProgressReady: + // Objects added during the scan can take the done count past the total. + current.ItemsTotal = max(current.ItemsTotal, current.ItemsDone) + current.BytesTotal = max(current.BytesTotal, current.BytesDone) + case scannedWholeBucket: + // The count pass did not get to the end, but the scan did, so what it did is the total. + current.ItemsTotal = current.ItemsDone + current.BytesTotal = current.BytesDone + current.State = sources.UnitProgressReady + default: + current.State = sources.UnitProgressUnavailable + } + p.finished = true + }) +} + +// startCount runs the count pass alongside the scan. It returns a function that stops the count and +// waits for it to exit, so nothing is counted after the unit finishes. +func (s *Source) startCount( + ctx context.Context, + client *s3.Client, + bucket string, + startAfter *string, + progress *unitProgress, +) func() { + countCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + + progress.setState(sources.UnitProgressCounting) + go func() { + defer close(done) + defer common.Recover(countCtx) + s.countBucket(countCtx, client, bucket, startAfter, progress) + }() + + return func() { + cancel() + <-done + } +} + +// countBucket lists the bucket from the start, the same way the scan does, to add up the objects the +// scan downloads and their size, since S3 has no API that reports either. Keys at or before startAfter +// were finished by an earlier run, so they count as done too. Nothing is kept per object. +func (s *Source) countBucket( + ctx context.Context, + client *s3.Client, + bucket string, + startAfter *string, + progress *unitProgress, +) { + paginator := newBucketPaginator(client, s.listInputs(bucket, nil)) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + // A cancelled count means the unit has finished, and finish settles the totals. + if ctx.Err() == nil { + ctx.Logger().V(2).Info("could not count objects for progress", "bucket", bucket, "err", err) + progress.setState(sources.UnitProgressUnavailable) + } + return + } + + var items, bytes, resumedItems, resumedBytes uint64 + for _, obj := range page.Contents { + if s.skipReason(obj) != "" { + continue + } + size := uint64(max(*obj.Size, 0)) + items++ + bytes += size + if startAfter != nil && *obj.Key <= *startAfter { + resumedItems++ + resumedBytes += size + } + } + progress.counted(items, bytes, resumedItems, resumedBytes) + } + progress.setState(sources.UnitProgressReady) +} diff --git a/pkg/sources/s3/progress_test.go b/pkg/sources/s3/progress_test.go new file mode 100644 index 000000000000..525450adbc43 --- /dev/null +++ b/pkg/sources/s3/progress_test.go @@ -0,0 +1,357 @@ +package s3 + +import ( + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + "github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest" +) + +func TestUnitProgress_CountThenScan(t *testing.T) { + var progress sources.Progress + p := newUnitProgress("unit", &progress) + + p.setState(sources.UnitProgressCounting) + p.counted(4, 4000, 1, 1000) + + got, ok := progress.GetUnitProgressFor("unit") + require.True(t, ok) + assert.Equal(t, sources.UnitProgress{ + State: sources.UnitProgressCounting, + ItemsDone: 1, + ItemsTotal: 4, + BytesDone: 1000, + BytesTotal: 4000, + ResumedBytes: 1000, + }, got) + assert.Zero(t, got.Percent(), "no percent while the total is still being counted") + + p.setState(sources.UnitProgressReady) + p.objectDone(1000) + + got, _ = progress.GetUnitProgressFor("unit") + assert.EqualValues(t, 2, got.ItemsDone) + assert.EqualValues(t, 2000, got.BytesDone) + assert.EqualValues(t, 50, got.Percent()) +} + +func TestUnitProgress_Finish(t *testing.T) { + tests := []struct { + name string + state sources.UnitProgressState + itemsTotal uint64 + itemsDone uint64 + scannedWholeBucket bool + wantState sources.UnitProgressState + wantItemsTotal uint64 + }{ + { + name: "counted, and objects added during the scan", + state: sources.UnitProgressReady, itemsTotal: 4, itemsDone: 5, scannedWholeBucket: true, + wantState: sources.UnitProgressReady, wantItemsTotal: 5, + }, + { + name: "counted, and the scan listing broke off", + state: sources.UnitProgressReady, itemsTotal: 4, itemsDone: 2, + wantState: sources.UnitProgressReady, wantItemsTotal: 4, + }, + { + name: "scan listed the whole bucket before the count finished", + state: sources.UnitProgressCounting, itemsTotal: 1, itemsDone: 3, scannedWholeBucket: true, + wantState: sources.UnitProgressReady, wantItemsTotal: 3, + }, + { + name: "count failed but the scan listed the whole bucket", + state: sources.UnitProgressUnavailable, itemsTotal: 1, itemsDone: 3, scannedWholeBucket: true, + wantState: sources.UnitProgressReady, wantItemsTotal: 3, + }, + { + name: "resumed scan finished before the count", + state: sources.UnitProgressCounting, itemsTotal: 1, itemsDone: 3, + wantState: sources.UnitProgressUnavailable, wantItemsTotal: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var progress sources.Progress + p := newUnitProgress("unit", &progress) + p.setState(tt.state) + p.counted(tt.itemsTotal, tt.itemsTotal*100, 0, 0) + for range tt.itemsDone { + p.objectDone(100) + } + + p.finish(tt.scannedWholeBucket) + + got, _ := progress.GetUnitProgressFor("unit") + assert.Equal(t, tt.wantState, got.State) + assert.Equal(t, tt.wantItemsTotal, got.ItemsTotal) + assert.Equal(t, tt.wantItemsTotal*100, got.BytesTotal) + assert.Equal(t, tt.itemsDone, got.ItemsDone) + }) + } +} + +func TestUnitProgress_IgnoresUpdatesAfterFinish(t *testing.T) { + var progress sources.Progress + p := newUnitProgress("unit", &progress) + p.objectDone(100) + p.finish(true) + want, _ := progress.GetUnitProgressFor("unit") + + p.objectDone(100) + p.counted(5, 500, 0, 0) + p.setState(sources.UnitProgressUnavailable) + + got, _ := progress.GetUnitProgressFor("unit") + assert.Equal(t, want, got) +} + +func TestUnitProgress_NilIsANoOp(t *testing.T) { + var p *unitProgress + assert.NotPanics(t, func() { p.objectDone(100) }) +} + +// fakeBucket is a bucket served by fakeS3. +type fakeBucket struct { + objects map[string]string + // countDelay holds back listings that start from the beginning of the bucket, which only the count + // makes when the scan is resumed, so a test can make the scan finish first. + countDelay time.Duration + // listPrefix denies listings outside it, as an IAM policy on s3:prefix does. + listPrefix string +} + +// fakeS3 serves the parts of the S3 API a unit scan uses, path style, two keys per page. +type fakeS3 struct { + buckets map[string]fakeBucket + getDelay time.Duration + listCalls atomic.Int32 +} + +func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + name, key, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + bucket, ok := f.buckets[name] + if !ok { + http.Error(w, "no such bucket", http.StatusNotFound) + return + } + + if key == "" { + f.list(w, r, name, bucket) + return + } + + if !sleep(r, f.getDelay) { + return + } + body, ok := bucket.objects[key] + if !ok { + http.Error(w, "no such key", http.StatusNotFound) + return + } + _, _ = w.Write([]byte(body)) +} + +func (f *fakeS3) list(w http.ResponseWriter, r *http.Request, name string, bucket fakeBucket) { + f.listCalls.Add(1) + const pageSize = 2 + + query := r.URL.Query() + prefix := query.Get("prefix") + if !strings.HasPrefix(prefix, bucket.listPrefix) { + http.Error(w, "access denied", http.StatusForbidden) + return + } + + after := query.Get("continuation-token") + if after == "" { + after = query.Get("start-after") + if after == "" && !sleep(r, bucket.countDelay) { + return + } + } + + keys := make([]string, 0, len(bucket.objects)) + for key := range bucket.objects { + if strings.HasPrefix(key, prefix) && key > after { + keys = append(keys, key) + } + } + slices.Sort(keys) + truncated := len(keys) > pageSize + if truncated { + keys = keys[:pageSize] + } + + var b strings.Builder + fmt.Fprintf(&b, ``+ + `%s%d1000%t`, name, len(keys), truncated) + for _, key := range keys { + fmt.Fprintf(&b, `%s2024-01-01T00:00:00.000Z%d`+ + `STANDARD`, key, len(bucket.objects[key])) + } + if truncated { + fmt.Fprintf(&b, `%s`, keys[len(keys)-1]) + } + b.WriteString(``) + + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(b.String())) +} + +// sleep waits for d unless the request is cancelled first, and reports whether it waited the whole time. +func sleep(r *http.Request, d time.Duration) bool { + if d == 0 { + return true + } + select { + case <-time.After(d): + return true + case <-r.Context().Done(): + return false + } +} + +// newFakeS3Source starts fake as an S3-compatible endpoint and returns a source initialized against it, +// after configure has adjusted the connection. +func newFakeS3Source(t *testing.T, fake *fakeS3, configure ...func(*sourcespb.S3)) *Source { + t.Helper() + server := httptest.NewServer(fake) + t.Cleanup(server.Close) + + s3Conn := &sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + Endpoint: server.URL, + } + for _, f := range configure { + f(s3Conn) + } + conn, err := anypb.New(s3Conn) + require.NoError(t, err) + + s := &Source{} + require.NoError(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1)) + return s +} + +func enableUnitProgress(t *testing.T) { + t.Helper() + feature.EnableS3UnitProgress.Store(true) + t.Cleanup(func() { feature.EnableS3UnitProgress.Store(false) }) +} + +// testObjects are three objects of 10, 20 and 30 bytes, and one the scan skips as empty. +func testObjects() map[string]string { + return map[string]string{ + "a.txt": strings.Repeat("a", 10), + "b.txt": strings.Repeat("b", 20), + "c.txt": strings.Repeat("c", 30), + "empty.txt": "", + } +} + +func TestSource_ChunkUnit_UnitProgress(t *testing.T) { + enableUnitProgress(t) + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: testObjects()}}} + s := newFakeS3Source(t, fake) + + unit := S3SourceUnit{Bucket: "bucket"} + reporter := sourcestest.TestReporter{} + require.NoError(t, s.ChunkUnit(context.Background(), unit, &reporter)) + + unitID, _ := unit.SourceUnitID() + got, ok := s.GetUnitProgressFor(unitID) + require.True(t, ok) + assert.Equal(t, sources.UnitProgress{ + State: sources.UnitProgressReady, + ItemsDone: 3, + ItemsTotal: 3, + BytesDone: 60, + BytesTotal: 60, + }, got) + assert.EqualValues(t, 99, got.Percent()) + assert.NotEmpty(t, reporter.Chunks) +} + +func TestSource_ChunkUnit_ResumedUnitProgress(t *testing.T) { + enableUnitProgress(t) + // Slow downloads let the count finish first. + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: testObjects()}}, getDelay: 200 * time.Millisecond} + s := newFakeS3Source(t, fake) + + unit := S3SourceUnit{Bucket: "bucket"} + unitID, _ := unit.SourceUnitID() + s.SetEncodedResumeInfoFor(unitID, "a.txt") + + require.NoError(t, s.ChunkUnit(context.Background(), unit, &sourcestest.TestReporter{})) + + got, _ := s.GetUnitProgressFor(unitID) + assert.Equal(t, sources.UnitProgress{ + State: sources.UnitProgressReady, + ItemsDone: 3, + ItemsTotal: 3, + BytesDone: 60, + BytesTotal: 60, + ResumedBytes: 10, + }, got, "the object an earlier run finished counts as done") +} + +// With one progress shared by every bucket, a count cut short in one bucket left the others at 0%. +// Each unit now has its own, so one bucket's count cannot affect another's. +func TestSource_ChunkUnit_UnitsDoNotShareProgress(t *testing.T) { + enableUnitProgress(t) + fake := &fakeS3{buckets: map[string]fakeBucket{ + "resumed": {objects: testObjects(), countDelay: 10 * time.Second}, + "fresh": {objects: testObjects()}, + }} + s := newFakeS3Source(t, fake) + + // The resumed scan finishes long before its count, which is cancelled. + resumed := S3SourceUnit{Bucket: "resumed"} + resumedID, _ := resumed.SourceUnitID() + s.SetEncodedResumeInfoFor(resumedID, "a.txt") + require.NoError(t, s.ChunkUnit(context.Background(), resumed, &sourcestest.TestReporter{})) + + fresh := S3SourceUnit{Bucket: "fresh"} + require.NoError(t, s.ChunkUnit(context.Background(), fresh, &sourcestest.TestReporter{})) + + got, _ := s.GetUnitProgressFor(resumedID) + assert.Equal(t, sources.UnitProgressUnavailable, got.State, "a resumed bucket with no count has no total") + assert.EqualValues(t, 2, got.ItemsDone) + assert.Zero(t, got.Percent()) + + freshID, _ := fresh.SourceUnitID() + got, _ = s.GetUnitProgressFor(freshID) + assert.Equal(t, sources.UnitProgressReady, got.State) + assert.EqualValues(t, 60, got.BytesTotal) + assert.EqualValues(t, 99, got.Percent()) +} + +func TestSource_ChunkUnit_NoUnitProgressUnlessEnabled(t *testing.T) { + fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: testObjects()}}} + s := newFakeS3Source(t, fake) + + unit := S3SourceUnit{Bucket: "bucket"} + require.NoError(t, s.ChunkUnit(context.Background(), unit, &sourcestest.TestReporter{})) + + unitID, _ := unit.SourceUnitID() + _, ok := s.GetUnitProgressFor(unitID) + assert.False(t, ok) + assert.EqualValues(t, 2, fake.listCalls.Load(), "only the scan lists the bucket") +} diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index 8b0939f2045f..9c8a821da4f7 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -1,8 +1,10 @@ package s3 import ( + "encoding/base64" "encoding/json" "fmt" + "net/url" "slices" "strings" "sync" @@ -24,6 +26,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" "github.com/trufflesecurity/trufflehog/v3/pkg/log" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" @@ -54,6 +57,10 @@ type Source struct { errorCount *sync.Map jobPool *errgroup.Group maxObjectSize int64 + // endpoint is the S3-compatible service to scan, or nil for AWS S3. + endpoint *url.URL + // objectFilter is never nil after Init. + objectFilter *objectFilter } // Ensure the Source satisfies the interfaces at compile time @@ -94,6 +101,12 @@ func (s *Source) Init( } s.conn = &conn + endpoint, err := normalizeEndpoint(conn.GetEndpoint()) + if err != nil { + return err + } + s.endpoint = endpoint + s.metricsCollector = metricsInstance s.setMaxObjectSize(conn.GetMaxObjectSize()) @@ -102,6 +115,25 @@ func (s *Source) Init( return errors.New("either a bucket include list or a bucket ignore list can be specified, but not both") } + filter, err := newObjectFilter( + conn.GetIncludePrefixes(), + conn.GetExcludePrefixes(), + conn.GetIncludeExtensions(), + conn.GetExcludeExtensions(), + ) + if err != nil { + return err + } + s.objectFilter = filter + + if filter.isConfigured() { + ctx.Logger().V(1).Info("Object filter configured", + "include_prefixes", filter.includePrefixes, + "exclude_prefixes", filter.excludePrefixes, + "include_extensions", filter.includeExtensions, + "exclude_extensions", filter.excludeExtensions) + } + return nil } @@ -133,6 +165,44 @@ func (s *Source) setMaxObjectSize(maxObjectSize int64) { } } +// normalizeEndpoint parses the configured endpoint of an S3-compatible +// service, returning nil for the empty endpoint that means AWS S3. A scheme is +// assumed when one is missing, because the endpoint is user-entered and the AWS +// SDK rejects a bare host with an error that does not say so. +func normalizeEndpoint(endpoint string) (*url.URL, error) { + if endpoint == "" { + return nil, nil + } + + parsed, err := url.Parse(endpoint) + + // Without a scheme, the endpoint either parses as a path with no host or, + // when it carries a port, fails to parse at all. Retry those as URLs, but + // only when no scheme was given, so that a malformed one is still reported + // rather than buried under a second scheme. + if !strings.Contains(endpoint, "://") && (err != nil || parsed.Host == "") { + parsed, err = url.Parse("https://" + endpoint) + } + if err != nil { + return nil, fmt.Errorf("could not parse endpoint %q: %w", endpoint, err) + } + + if parsed.Host == "" { + return nil, fmt.Errorf("endpoint %q has no host; expected something like https://s3.internal.example.com", endpoint) + } + + return parsed, nil +} + +// defaultRegion returns the region used to sign requests for buckets whose own +// region has not been discovered. +func (s *Source) defaultRegion() string { + if region := s.conn.GetRegion(); region != "" { + return region + } + return defaultAWSRegion +} + func (s *Source) newClient(ctx context.Context, region, roleArn string) (*s3.Client, error) { var credsProvider aws.CredentialsProvider switch cred := s.conn.GetCredential().(type) { @@ -182,6 +252,12 @@ func (s *Source) newClient(ctx context.Context, region, roleArn string) (*s3.Cli return s3.NewFromConfig(cfg, func(options *s3.Options) { options.DisableLogOutputChecksumValidationSkipped = true + if s.endpoint != nil { + options.BaseEndpoint = aws.String(s.endpoint.String()) + // S3-compatible services rarely publish the wildcard DNS that + // virtual-hosted addressing needs. + options.UsePathStyle = true + } }), nil } @@ -231,8 +307,10 @@ type pageMetadata struct { // processingState tracks the state of concurrent S3 object processing. type processingState struct { - errorCount *sync.Map // Thread-safe map tracking errors per prefix - objectCount *uint64 // Total number of objects processed + errorCount *sync.Map // Thread-safe map tracking errors per prefix + objectCount *uint64 // Total number of objects processed + filteredCount *uint64 // Total number of objects excluded by the object filter + progress *unitProgress // Progress inside the unit, nil for a legacy scan } // resumePosition tracks where to restart scanning S3 buckets and objects after an interruption. @@ -285,17 +363,21 @@ func determineResumePosition(ctx context.Context, tracker *Checkpointer, buckets } } +// scanBuckets scans the given buckets using the given role and adds the number +// of objects it scanned to totalObjectCount. The counter is owned by Chunks and +// shared across role passes so that the completion message reflects the whole +// scan, not just the last role's pass. func (s *Source) scanBuckets( ctx context.Context, client *s3.Client, role string, bucketsToScan []string, chunksChan chan *sources.Chunk, + totalObjectCount *uint64, ) { if role != "" { ctx = context.WithValue(ctx, "role", role) } - var totalObjectCount uint64 checkpointer := NewCheckpointer(ctx, &s.Progress, false) pos := determineResumePosition(ctx, checkpointer, bucketsToScan) @@ -326,7 +408,7 @@ func (s *Source) scanBuckets( bucketIdx, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), - s.Progress.EncodedResumeInfo, + s.EncodedResumeInfo, ) var startAfter *string @@ -339,14 +421,14 @@ func (s *Source) scanBuckets( ) } - objectCount := s.scanBucket(ctx, client, role, bucket, sources.ChanReporter{Ch: chunksChan}, startAfter, checkpointer) - totalObjectCount += objectCount + objectCount := s.scanBucket(ctx, client, role, bucket, sources.ChanReporter{Ch: chunksChan}, startAfter, checkpointer, nil) + *totalObjectCount += objectCount } s.SetProgressComplete( len(bucketsToScan), len(bucketsToScan), - fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, totalObjectCount), + fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, *totalObjectCount), "", ) } @@ -359,6 +441,7 @@ func (s *Source) scanBucket( reporter sources.ChunkReporter, startAfter *string, checkpointer *Checkpointer, + progress *unitProgress, ) uint64 { s.metricsCollector.RecordBucketForRole(role) @@ -377,28 +460,41 @@ func (s *Source) scanBucket( return 0 } - errorCount := sync.Map{} - - input := &s3.ListObjectsV2Input{Bucket: &bucket} - if startAfter != nil { - input.StartAfter = startAfter + // scannedWholeBucket is set once the listing below has gone from start to end, so it has seen every + // object the scan can include. + var scannedWholeBucket bool + if progress != nil { + stopCount := s.startCount(ctx, regionalClient, bucket, startAfter, progress) + defer func() { + stopCount() + progress.finish(scannedWholeBucket) + }() } + errorCount := sync.Map{} + pageNumber := 1 - paginator := s3.NewListObjectsV2Paginator(regionalClient, input) - var objectCount uint64 + paginator := newBucketPaginator(regionalClient, s.listInputs(bucket, startAfter)) + var objectCount, filteredCount uint64 + var listed int + listFailed := false for paginator.HasMorePages() { output, err := paginator.NextPage(ctx) if err != nil { - if role == "" { - ctx.Logger().Error(err, "could not list objects in bucket") - } else { + if s.listErrorsAreExpected(role) { // Our documentation blesses specifying a role to assume without specifying buckets to scan, which will // often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the // account, but the role probably doesn't have access to all of them). This makes it expected behavior // and therefore not an error. ctx.Logger().V(3).Info("could not list objects in bucket", "err", err) + } else { + // This can also be a failure to assume the role itself: role credentials + // are resolved lazily, so the first request that needs them surfaces the + // STS error here rather than at client construction. + ctx.Logger().Error(err, "could not list objects in bucket") } + s.metricsCollector.RecordBucketListError(bucket, role) + listFailed = true break } pageMetadata := pageMetadata{ @@ -409,20 +505,47 @@ func (s *Source) scanBucket( page: output, } processingState := processingState{ - errorCount: &errorCount, - objectCount: &objectCount, + errorCount: &errorCount, + objectCount: &objectCount, + filteredCount: &filteredCount, + progress: progress, } s.pageChunker(ctx, pageMetadata, processingState, reporter, checkpointer) + listed += len(output.Contents) pageNumber++ } + scannedWholeBucket = startAfter == nil && !listFailed && ctx.Err() == nil + + // A filter that excludes everything otherwise looks exactly like a clean scan + // of an empty bucket, so say so rather than finishing silently. + if objectCount == 0 && filteredCount > 0 { + ctx.Logger().Info("Scanned no objects in bucket", "excluded_by_object_filter", filteredCount) + } + // The same goes for include prefixes, which S3 applies itself: a mistyped one lists nothing rather than + // excluding anything. + if prefixes := s.objectFilter.listPrefixes(); len(prefixes) > 0 && listed == 0 && scannedWholeBucket { + ctx.Logger().Info("Found no objects under the include prefixes", "include_prefixes", prefixes) + } + return objectCount } +// listErrorsAreExpected reports whether a failure to list a bucket's objects +// should be suppressed rather than logged as an error. When a role is assumed +// without an explicit bucket list, the scanner attempts every bucket in the +// account and is expected to be denied on some of them. When buckets are +// explicitly configured, a listing failure means a configured target is being +// silently skipped, so it is always an error, even under an assumed role. +func (s *Source) listErrorsAreExpected(role string) bool { + return role != "" && len(s.conn.GetBuckets()) == 0 +} + // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error { + var totalObjectCount uint64 visitor := func(c context.Context, defaultRegionClient *s3.Client, roleArn string, buckets []string) error { - s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan) + s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan, &totalObjectCount) return nil } @@ -435,12 +558,18 @@ func (s *Source) getRegionalClientForBucket( role string, bucket string, ) (*s3.Client, error) { + // GetBucketRegion is an AWS-only API, and a custom endpoint serves every + // bucket itself, so there is no per-bucket region to discover. + if s.endpoint != nil { + return defaultRegionClient, nil + } + region, err := s3manager.GetBucketRegion(ctx, defaultRegionClient, bucket) if err != nil { return nil, fmt.Errorf("could not get s3 region for bucket: %s: %w", bucket, err) } - if region == defaultAWSRegion { + if region == s.defaultRegion() { return defaultRegionClient, nil } @@ -452,6 +581,35 @@ func (s *Source) getRegionalClientForBucket( return regionalClient, nil } +// Reasons the scan skips an object without downloading it. They are also the reason label on the +// skipped objects metric. +const ( + skipReasonObjectFilter = "object_filter" + skipReasonStorageClass = "storage_class" + skipReasonSizeLimit = "size_limit" + skipReasonEmptyFile = "empty_file" + skipReasonIncompatibleExtension = "incompatible_extension" +) + +// skipReason returns why the scan skips an object without downloading it, or "" if it downloads it. +// The count pass uses it too, so the progress total holds exactly the objects the scan downloads. +func (s *Source) skipReason(obj s3types.Object) string { + switch { + case !s.objectFilter.shouldInclude(*obj.Key): + return skipReasonObjectFilter + case obj.StorageClass == s3types.ObjectStorageClassGlacier || obj.StorageClass == s3types.ObjectStorageClassGlacierIr: + return skipReasonStorageClass + case *obj.Size > s.maxObjectSize: + return skipReasonSizeLimit + case *obj.Size == 0: + return skipReasonEmptyFile + case common.SkipFile(*obj.Key): + return skipReasonIncompatibleExtension + default: + return "" + } +} + // pageChunker emits chunks onto the given channel from a page. func (s *Source) pageChunker( ctx context.Context, @@ -463,59 +621,34 @@ func (s *Source) pageChunker( checkpointer.Reset() // Reset the checkpointer for each PAGE ctx = context.WithValues(ctx, "bucket", metadata.bucket, "page_number", metadata.pageNumber) for objIdx, obj := range metadata.page.Contents { - ctx = context.WithValues(ctx, "key", *obj.Key, "size", *obj.Size) - if common.IsDone(ctx) { + octx := context.WithValues(ctx, "key", *obj.Key, "size", *obj.Size) + if common.IsDone(octx) { return } - // Skip GLACIER and GLACIER_IR objects. - if obj.StorageClass == s3types.ObjectStorageClassGlacier || obj.StorageClass == s3types.ObjectStorageClassGlacierIr { - ctx.Logger().V(5).Info("Skipping object in storage class", "storage_class", obj.StorageClass) - s.metricsCollector.RecordObjectSkipped(metadata.bucket, "storage_class", float64(*obj.Size)) - if err := checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { - ctx.Logger().Error(err, "could not update progress for glacier object") - } - continue - } - - // Ignore large files. - if *obj.Size > s.maxObjectSize { - ctx.Logger().V(5).Info("Skipping large file", "max_object_size", s.maxObjectSize) - s.metricsCollector.RecordObjectSkipped(metadata.bucket, "size_limit", float64(*obj.Size)) - if err := checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { - ctx.Logger().Error(err, "could not update progress for large file") - } - continue - } - - // File empty file. - if *obj.Size == 0 { - ctx.Logger().V(5).Info("Skipping empty file") - s.metricsCollector.RecordObjectSkipped(metadata.bucket, "empty_file", 0) - if err := checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { - ctx.Logger().Error(err, "could not update progress for empty file") + // Skip objects the scan never downloads: filtered, Glacier, too large, empty or of an + // incompatible extension. They are left out of the unit's progress on both sides. + if reason := s.skipReason(obj); reason != "" { + if reason == skipReasonObjectFilter { + atomic.AddUint64(state.filteredCount, 1) } - continue - } - - // Skip incompatible extensions. - if common.SkipFile(*obj.Key) { - ctx.Logger().V(5).Info("Skipping file with incompatible extension") - s.metricsCollector.RecordObjectSkipped(metadata.bucket, "incompatible_extension", float64(*obj.Size)) - if err := checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { - ctx.Logger().Error(err, "could not update progress for incompatible file") + octx.Logger().V(5).Info("Skipping object", "reason", reason) + s.metricsCollector.RecordObjectSkipped(metadata.bucket, reason, float64(*obj.Size)) + if err := checkpointer.UpdateObjectCompletion(octx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { + octx.Logger().Error(err, "could not update progress for skipped object", "reason", reason) } continue } s.jobPool.Go(func() error { - defer common.RecoverWithExit(ctx) - if common.IsDone(ctx) { - return ctx.Err() + defer common.RecoverWithExit(octx) + if common.IsDone(octx) { + return octx.Err() } + defer state.progress.objectDone(*obj.Size) if strings.HasSuffix(*obj.Key, "/") { - ctx.Logger().V(5).Info("Skipping directory") + octx.Logger().V(5).Info("Skipping directory") s.metricsCollector.RecordObjectSkipped(metadata.bucket, "directory", float64(*obj.Size)) return nil } @@ -528,13 +661,13 @@ func (s *Source) pageChunker( nErr = 0 } if nErr.(int) > 3 { - ctx.Logger().V(2).Info("Skipped due to excessive errors") + octx.Logger().V(2).Info("Skipped due to excessive errors") return nil } // Make sure we use a separate context for the GetObjectWithContext call. // This ensures that the timeout is isolated and does not affect any downstream operations. (e.g. HandleFile) const getObjectTimeout = 30 * time.Second - objCtx, cancel := context.WithTimeout(ctx, getObjectTimeout) + objCtx, cancel := context.WithTimeout(octx, getObjectTimeout) defer cancel() res, err := metadata.client.GetObject(objCtx, &s3.GetObjectInput{ @@ -543,10 +676,10 @@ func (s *Source) pageChunker( }) if err != nil { if strings.Contains(err.Error(), "AccessDenied") { - ctx.Logger().Error(err, "could not get S3 object; access denied") + octx.Logger().Error(err, "could not get S3 object; access denied") s.metricsCollector.RecordObjectSkipped(metadata.bucket, "access_denied", float64(*obj.Size)) } else { - ctx.Logger().Error(err, "could not get S3 object") + octx.Logger().Error(err, "could not get S3 object") s.metricsCollector.RecordObjectError(metadata.bucket) } // According to the documentation for GetObjectWithContext, @@ -554,7 +687,7 @@ func (s *Source) pageChunker( // It's uncertain if the body will be nil in such cases, // but we'll close it if it's not. if res != nil && res.Body != nil { - res.Body.Close() + _ = res.Body.Close() } nErr, ok := state.errorCount.Load(prefix) @@ -562,18 +695,18 @@ func (s *Source) pageChunker( nErr = 0 } if nErr.(int) > 3 { - ctx.Logger().V(3).Info("Skipped due to excessive errors") + octx.Logger().V(3).Info("Skipped due to excessive errors") return nil } nErr = nErr.(int) + 1 state.errorCount.Store(prefix, nErr) // too many consecutive errors on this page if nErr.(int) > 3 { - ctx.Logger().V(2).Info("Too many consecutive errors, excluding prefix", "prefix", prefix) + octx.Logger().V(2).Info("Too many consecutive errors, excluding prefix", "prefix", prefix) } return nil } - defer res.Body.Close() + defer func() { _ = res.Body.Close() }() email := "Unknown" if obj.Owner != nil { @@ -590,7 +723,7 @@ func (s *Source) pageChunker( S3: &source_metadatapb.S3{ Bucket: metadata.bucket, File: sanitizer.UTF8(*obj.Key), - Link: sanitizer.UTF8(makeS3Link(metadata.bucket, metadata.client.Options().Region, *obj.Key)), + Link: sanitizer.UTF8(s.objectLink(metadata.bucket, metadata.client.Options().Region, *obj.Key)), Email: sanitizer.UTF8(email), Timestamp: sanitizer.UTF8(modified), }, @@ -599,13 +732,13 @@ func (s *Source) pageChunker( SourceVerify: s.verify, } - if err := handlers.HandleFile(ctx, res.Body, chunkSkel, reporter); err != nil { - ctx.Logger().Error(err, "error handling file") + if err := handlers.HandleFile(octx, res.Body, chunkSkel, reporter); err != nil { + octx.Logger().Error(err, "error handling file") s.metricsCollector.RecordObjectError(metadata.bucket) return nil } atomic.AddUint64(state.objectCount, 1) - ctx.Logger().V(5).Info("S3 object scanned.", "object_count", state.objectCount) + octx.Logger().V(5).Info("S3 object scanned.", "object_count", state.objectCount) nErr, ok = state.errorCount.Load(prefix) if !ok { nErr = 0 @@ -614,8 +747,8 @@ func (s *Source) pageChunker( state.errorCount.Store(prefix, 0) } // Update progress after successful processing. - if err := checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { - ctx.Logger().Error(err, "could not update progress for scanned object") + if err := checkpointer.UpdateObjectCompletion(octx, objIdx, metadata.bucket, metadata.role, metadata.page.Contents); err != nil { + octx.Logger().Error(err, "could not update progress for scanned object") } s.metricsCollector.RecordObjectScanned(metadata.bucket, float64(*obj.Size)) return nil @@ -640,7 +773,8 @@ func (s *Source) validateBucketAccess(ctx context.Context, client *s3.Client, ro continue } - _, err = regionalClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) + // Make the scan's first listing, so access limited to the include prefixes passes. + _, err = regionalClient.ListObjectsV2(ctx, s.listInputs(bucket, nil)[0]) if err == nil { wasAbleToListAnyBucket = true } else if shouldHaveAccessToAllBuckets { @@ -679,7 +813,7 @@ func (s *Source) visitRoles( for _, role := range roles { s.metricsCollector.RecordRoleScanned(role) - client, err := s.newClient(ctx, defaultAWSRegion, role) + client, err := s.newClient(ctx, s.defaultRegion(), role) if err != nil { return fmt.Errorf("could not create s3 client: %w", err) } @@ -697,12 +831,22 @@ func (s *Source) visitRoles( return nil } -// makeS3Link creates a S3 virtual-hosted–style URIs. They have the format of: +// objectLink creates a URL for an object. AWS buckets get a +// virtual-hosted–style URI, which has the format: // https://[bucket-name].s3.[region-code].amazonaws.com/[key-name] // // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access -func makeS3Link(bucket, region, key string) string { - return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, key) +// +// A custom endpoint gets a path-style link, matching how the client addresses +// it, so the link resolves the same way the scan did. +func (s *Source) objectLink(bucket, region, key string) string { + if s.endpoint == nil { + return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, key) + } + + link := *s.endpoint + link.Path = strings.TrimSuffix(link.Path, "/") + "/" + bucket + "/" + key + return link.String() } // Enumerate implements SourceUnitEnumerator interface. This implementation visits @@ -740,7 +884,7 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte } // unitID is a combination of bucket name and role ARN unitID, _ := s3unit.SourceUnitID() - defaultClient, err := s.newClient(ctx, defaultAWSRegion, s3unit.Role) + defaultClient, err := s.newClient(ctx, s.defaultRegion(), s3unit.Role) if err != nil { return fmt.Errorf("could not create s3 client for bucket %s and role %s: %w", s3unit.Bucket, s3unit.Role, err) } @@ -748,7 +892,7 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte checkpointer := NewCheckpointer(ctx, &s.Progress, true) var startAfterPtr *string - startAfter := s.Progress.GetEncodedResumeInfoFor(unitID) + startAfter := s.GetEncodedResumeInfoFor(unitID) if startAfter != "" { ctx.Logger().V(3).Info( "Resuming unit scan", @@ -757,12 +901,38 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte ) startAfterPtr = &startAfter } - defer s.Progress.ClearEncodedResumeInfoFor(unitID) - s.scanBucket(ctx, defaultClient, s3unit.Role, s3unit.Bucket, reporter, startAfterPtr, checkpointer) + defer s.ClearEncodedResumeInfoFor(unitID) + + // Counting the bucket costs an extra listing of it, so it only runs where something reads the result. + var progress *unitProgress + if feature.EnableS3UnitProgress.Load() { + progress = newUnitProgress(unitID, &s.Progress) + } + s.scanBucket(ctx, defaultClient, s3unit.Role, s3unit.Bucket, reporter, startAfterPtr, checkpointer, progress) return nil } +// It accepts three shapes: the persisted envelope carrying unit_data +// (round-trips the unit exactly), the envelope without unit_data (rebuilt from id), +// and a bare S3SourceUnit. func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) { + var envelope unitEnvelope + if err := json.Unmarshal(data, &envelope); err == nil && envelope.Kind == string(SourceUnitKindBucket) { + if envelope.UnitData != "" { + if decoded, err := base64.StdEncoding.DecodeString(envelope.UnitData); err == nil { + var unit S3SourceUnit + if json.Unmarshal(decoded, &unit) == nil && unit.Bucket != "" { + return unit, nil + } + } + } + if envelope.ID != "" { + if role, bucket := splitS3SourceUnitID(envelope.ID); bucket != "" { + return S3SourceUnit{Bucket: bucket, Role: role}, nil + } + } + } + var unit S3SourceUnit if err := json.Unmarshal(data, &unit); err != nil { return nil, err diff --git a/pkg/sources/s3/s3_integration_test.go b/pkg/sources/s3/s3_integration_test.go index 002864ce8ec1..43aa1fc0c775 100644 --- a/pkg/sources/s3/s3_integration_test.go +++ b/pkg/sources/s3/s3_integration_test.go @@ -644,3 +644,34 @@ func TestSourceChunksResumptionWithRole(t *testing.T) { assert.Equal(t, 9638, count, "Should have processed all remaining data on resume") assert.Less(t, count, sourceTotalChunkCount, "Should have processed less than total chunks on resume") } + +func TestSource_ChunkUnit_ReportsUnitProgress(t *testing.T) { + enableUnitProgress(t) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + Buckets: []string{"truffletestbucket"}, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(ctx, "test percent", 0, 0, false, conn, 1)) + + reporter := sourcestest.TestReporter{} + require.NoError(t, s.Enumerate(ctx, &reporter)) + require.Len(t, reporter.Units, 1) + require.NoError(t, s.ChunkUnit(ctx, reporter.Units[0], &reporter)) + + unitID, _ := reporter.Units[0].SourceUnitID() + progress, ok := s.GetUnitProgressFor(unitID) + require.True(t, ok) + assert.Equal(t, sources.UnitProgressReady, progress.State) + assert.Positive(t, progress.ItemsTotal) + assert.Equal(t, progress.ItemsTotal, progress.ItemsDone) + assert.Equal(t, progress.BytesTotal, progress.BytesDone) + assert.EqualValues(t, 99, progress.Percent()) + assert.Empty(t, s.GetEncodedResumeInfoFor(unitID)) +} diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go index d571e45a288e..5f5ecb64000f 100644 --- a/pkg/sources/s3/s3_test.go +++ b/pkg/sources/s3/s3_test.go @@ -2,11 +2,16 @@ package s3 import ( "encoding/base64" + "encoding/json" "fmt" "os" + "sync" "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/kylelemons/godebug/pretty" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,6 +43,76 @@ func TestSource_Init_IncludeAndIgnoreBucketsError(t *testing.T) { assert.Error(t, err) } +func TestSource_ListErrorsAreExpected(t *testing.T) { + tests := []struct { + name string + role string + buckets []string + want bool + }{ + { + name: "no role, no explicit buckets", + want: false, + }, + { + name: "no role, explicit buckets", + buckets: []string{"bucket-a"}, + want: false, + }, + { + name: "role without explicit buckets, denials are expected", + role: "arn:aws:iam::123456789012:role/some-role", + want: true, + }, + { + name: "role with explicit buckets, denials are errors", + role: "arn:aws:iam::123456789012:role/some-role", + buckets: []string{"bucket-a"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_AccessKey{ + AccessKey: &credentialspb.KeySecret{ + Key: "ignored for test", + Secret: "ignored for test", + }, + }, + Buckets: tt.buckets, + Roles: []string{tt.role}, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1)) + + assert.Equal(t, tt.want, s.listErrorsAreExpected(tt.role)) + }) + } +} + +func TestSource_ScanBucketsReportsCumulativeObjectCount(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1)) + + // Simulate a later role pass after an earlier pass already scanned three + // objects. The pass below scans no buckets, so the completion message must + // still report the cumulative total rather than resetting to zero. + totalObjectCount := uint64(3) + s.scanBuckets(context.Background(), nil, "", nil, make(chan *sources.Chunk, 1), &totalObjectCount) + + assert.Equal(t, uint64(3), totalObjectCount) + assert.Contains(t, s.Message, "3 objects scanned") +} + func TestSource_Chunks(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -166,19 +241,410 @@ func TestSource_Chunks(t *testing.T) { } func TestSource_UnmarshalSourceUnit(t *testing.T) { + roleARN := "arn:aws:iam::123456789012:role/my-role" + + // envelope mirrors the JSON shape thog persists between the enumerate and + // scan passes: the SourceUnit proto marshalled with encoding/json, where + // unit_data (if present) is the original unit as base64-encoded bytes. + type envelope struct { + ID string `json:"id"` + Kind string `json:"kind,omitempty"` + Display string `json:"display,omitempty"` + UnitData string `json:"unit_data,omitempty"` + } + + marshalEnvelope := func(t *testing.T, id string, kind string, unit *S3SourceUnit) []byte { + t.Helper() + env := envelope{ID: id, Kind: kind, Display: id} + if unit != nil { + raw, err := json.Marshal(unit) + require.NoError(t, err) + env.UnitData = base64.StdEncoding.EncodeToString(raw) + } + data, err := json.Marshal(env) + require.NoError(t, err) + return data + } + + tests := []struct { + name string + data []byte + wantUnit S3SourceUnit + wantErr bool + }{ + { + name: "bare unit, role-bearing ARN", + data: []byte(`{"Bucket":"my-test-bucket","Role":"` + roleARN + `"}`), + wantUnit: S3SourceUnit{Bucket: "my-test-bucket", Role: roleARN}, + }, + { + name: "bare unit, role-less", + data: []byte(`{"Bucket":"my-test-bucket"}`), + wantUnit: S3SourceUnit{Bucket: "my-test-bucket"}, + }, + { + name: "envelope with unit_data, role-bearing", + data: marshalEnvelope(t, constructS3SourceUnitID("my-test-bucket", roleARN), "bucket", + &S3SourceUnit{Bucket: "my-test-bucket", Role: roleARN}), + wantUnit: S3SourceUnit{Bucket: "my-test-bucket", Role: roleARN}, + }, + { + name: "envelope with unit_data, role-less", + data: marshalEnvelope(t, constructS3SourceUnitID("my-test-bucket", ""), "bucket", + &S3SourceUnit{Bucket: "my-test-bucket"}), + wantUnit: S3SourceUnit{Bucket: "my-test-bucket"}, + }, + { + name: "envelope without unit_data, role-bearing, rebuilt from id", + data: marshalEnvelope(t, constructS3SourceUnitID("my-test-bucket", roleARN), "bucket", nil), + wantUnit: S3SourceUnit{Bucket: "my-test-bucket", Role: roleARN}, + }, + { + name: "envelope without unit_data, role-less, rebuilt from id", + data: marshalEnvelope(t, constructS3SourceUnitID("my-test-bucket", ""), "bucket", nil), + wantUnit: S3SourceUnit{Bucket: "my-test-bucket"}, + }, + { + // A CommonSourceUnit-shaped bare unit from another source also has + // an "id" field; only S3's own "bucket" kind may trigger the + // envelope id-rebuild path, or this would be misread as a bucket. + name: "bare unit from another source is rejected, not misread as an envelope", + data: []byte(`{"kind":"repository","id":"some-repo-id"}`), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Source{} + unit, err := s.UnmarshalSourceUnit(tt.data) + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err, "UnmarshalSourceUnit should not return an error") + + s3Unit, ok := unit.(S3SourceUnit) + require.True(t, ok, "Unmarshaled unit should be of type S3SourceUnit") + assert.Equal(t, tt.wantUnit, s3Unit) + }) + } +} + +func TestSource_ObjectLink(t *testing.T) { + tests := []struct { + name string + endpoint string + want string + }{ + { + name: "aws", + want: "https://my-bucket.s3.eu-west-1.amazonaws.com/dir/key.txt", + }, + { + name: "custom endpoint", + endpoint: "https://s3.internal.example.com", + want: "https://s3.internal.example.com/my-bucket/dir/key.txt", + }, + { + name: "custom endpoint with trailing slash", + endpoint: "https://s3.internal.example.com/", + want: "https://s3.internal.example.com/my-bucket/dir/key.txt", + }, + { + name: "custom endpoint with port and base path", + endpoint: "http://minio.test:9000/base", + want: "http://minio.test:9000/base/my-bucket/dir/key.txt", + }, + { + name: "endpoint without a scheme is assumed https", + endpoint: "s3.internal.example.com", + want: "https://s3.internal.example.com/my-bucket/dir/key.txt", + }, + { + name: "host and port without a scheme is assumed https", + endpoint: "s3.internal.example.com:9000", + want: "https://s3.internal.example.com:9000/my-bucket/dir/key.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + Endpoint: tt.endpoint, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1)) + + assert.Equal(t, tt.want, s.objectLink("my-bucket", "eu-west-1", "dir/key.txt")) + }) + } +} + +func TestSource_Init_InvalidEndpoint(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + Endpoint: "https://s3.internal.example.com:not-a-port", + }) + require.NoError(t, err) + s := Source{} + assert.ErrorContains(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1), "endpoint") +} - unitJSON := `{ - "Bucket": "my-test-bucket", - "Role": "my-test-role" - }` +func TestSource_ClientAddressing(t *testing.T) { + tests := []struct { + name string + conn *sourcespb.S3 + wantRegion string + wantEndpoint *string + wantPathStyle bool + }{ + { + name: "aws defaults", + conn: &sourcespb.S3{}, + wantRegion: defaultAWSRegion, + }, + { + name: "explicit region without endpoint", + conn: &sourcespb.S3{Region: "ap-south-1"}, + wantRegion: "ap-south-1", + }, + { + name: "custom endpoint implies path style", + conn: &sourcespb.S3{Endpoint: "https://s3.internal.example.com"}, + wantRegion: defaultAWSRegion, + wantEndpoint: aws.String("https://s3.internal.example.com"), + wantPathStyle: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.conn.Credential = &sourcespb.S3_Unauthenticated{} + conn, err := anypb.New(tt.conn) + require.NoError(t, err) - unit, err := s.UnmarshalSourceUnit([]byte(unitJSON)) - require.NoError(t, err, "UnmarshalSourceUnit should not return an error") + s := Source{} + require.NoError(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1)) + + client, err := s.newClient(context.Background(), s.defaultRegion(), "") + require.NoError(t, err) + + opts := client.Options() + assert.Equal(t, tt.wantRegion, opts.Region) + assert.Equal(t, tt.wantEndpoint, opts.BaseEndpoint) + assert.Equal(t, tt.wantPathStyle, opts.UsePathStyle) + }) + } +} - s3Unit, ok := unit.(S3SourceUnit) - require.True(t, ok, "Unmarshaled unit should be of type S3SourceUnit") +func TestSource_Init_IncludeAndExcludeExtensionsError(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + IncludeExtensions: []string{"tf"}, + ExcludeExtensions: []string{"zip"}, + }) + assert.NoError(t, err) + + s := Source{} + err = s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1) - assert.Equal(t, "my-test-bucket", s3Unit.Bucket, "Bucket field should match") - assert.Equal(t, "my-test-role", s3Unit.Role, "Role field should match") + assert.Error(t, err) +} + +// Prefixes, unlike extensions and unlike buckets, accept an include list and an +// exclude list together. +func TestSource_Init_IncludeAndExcludePrefixesAllowed(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + IncludePrefixes: []string{"src/"}, + ExcludePrefixes: []string{"src/vendor/"}, + }) + assert.NoError(t, err) + + s := Source{} + err = s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1) + + assert.NoError(t, err) + assert.True(t, s.objectFilter.shouldInclude("src/main.tf")) + assert.False(t, s.objectFilter.shouldInclude("src/vendor/dep.tf")) +} + +// A source with no filters configured must scan every object, so that existing +// scans behave exactly as they did before object filtering was added. +func TestSource_Init_UnconfiguredFilterScansEverything(t *testing.T) { + conn, err := anypb.New(&sourcespb.S3{Credential: &sourcespb.S3_Unauthenticated{}}) + assert.NoError(t, err) + + s := Source{} + err = s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1) + + assert.NoError(t, err) + assert.True(t, s.objectFilter.shouldInclude("any/key.zip")) +} + +// A filtered object must still be marked complete on the checkpointer. Otherwise +// the low water mark stalls at the first filtered object and a resumed scan redoes +// every object after it. +func TestSource_PageChunker_FilteredObjectsAdvanceCheckpoint(t *testing.T) { + ctx := context.Background() + + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + ExcludePrefixes: []string{"archive/"}, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(ctx, "s3 test source", 0, 0, false, conn, 1)) + + const objectCount = 10 + page := &awss3.ListObjectsV2Output{Contents: make([]s3types.Object, objectCount)} + for i := range objectCount { + key := fmt.Sprintf("archive/key-%02d.txt", i) + size := int64(1024) + page.Contents[i] = s3types.Object{Key: &key, Size: &size} + } + + checkpointer := NewCheckpointer(ctx, &sources.Progress{}, false) + + // Every object is filtered, so pageChunker never reaches GetObject and needs + // no client. + var scanned, filtered uint64 + s.pageChunker( + ctx, + pageMetadata{bucket: "test-bucket", pageNumber: 1, page: page}, + processingState{errorCount: &sync.Map{}, objectCount: &scanned, filteredCount: &filtered}, + sources.ChanReporter{Ch: make(chan *sources.Chunk, objectCount)}, + checkpointer, + ) + + assert.Zero(t, scanned) + assert.EqualValues(t, objectCount, filtered) + + resumeInfo, err := checkpointer.ResumePoint(ctx) + require.NoError(t, err) + assert.Equal(t, "test-bucket", resumeInfo.CurrentBucket) + assert.Equal(t, *page.Contents[objectCount-1].Key, resumeInfo.StartAfter) +} + +func TestSource_PageChunker_UnitScanLeavesSkippedObjectsOutOfProgress(t *testing.T) { + ctx := context.Background() + + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + ExcludePrefixes: []string{"archive/"}, + MaxObjectSize: 1024, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(ctx, "s3 test source", 0, 0, false, conn, 1)) + s.SetEncodedResumeInfoFor("test-bucket", "earlier-key") + + filteredKey, glacierKey, oversizeKey, emptyKey, imageKey := "archive/a.txt", "cold.txt", "huge.txt", "none.txt", "photo.jpg" + size, oversize, empty := int64(512), int64(4096), int64(0) + page := &awss3.ListObjectsV2Output{Contents: []s3types.Object{ + {Key: &filteredKey, Size: &size}, + {Key: &glacierKey, Size: &size, StorageClass: s3types.ObjectStorageClassGlacier}, + {Key: &oversizeKey, Size: &oversize}, + {Key: &emptyKey, Size: &empty}, + {Key: &imageKey, Size: &size}, + }} + + // No object is downloaded, so pageChunker needs no client. + var scanned, filtered uint64 + s.pageChunker( + ctx, + pageMetadata{bucket: "test-bucket", pageNumber: 1, page: page}, + processingState{ + errorCount: &sync.Map{}, + objectCount: &scanned, + filteredCount: &filtered, + progress: newUnitProgress("test-bucket", &s.Progress), + }, + sources.ChanReporter{Ch: make(chan *sources.Chunk, len(page.Contents))}, + NewCheckpointer(ctx, &s.Progress, true), + ) + + _, reported := s.GetUnitProgressFor("test-bucket") + assert.False(t, reported, "skipped objects are in neither the total nor the done count") + assert.EqualValues(t, 1, filtered) + // Skipped objects still advance the checkpoint to the last key of the page. + assert.Equal(t, *page.Contents[len(page.Contents)-1].Key, s.GetEncodedResumeInfoFor("test-bucket")) +} + +func TestSource_PageChunker_LegacyScanLeavesProgressUntouched(t *testing.T) { + ctx := context.Background() + + conn, err := anypb.New(&sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + ExcludePrefixes: []string{"archive/"}, + }) + require.NoError(t, err) + + s := Source{} + require.NoError(t, s.Init(ctx, "s3 test source", 0, 0, false, conn, 1)) + s.SetProgressComplete(0, 1, "Bucket: test-bucket", "") + + key, size := "archive/key.txt", int64(1024) + page := &awss3.ListObjectsV2Output{Contents: []s3types.Object{{Key: &key, Size: &size}}} + + var scanned, filtered uint64 + s.pageChunker( + ctx, + pageMetadata{bucket: "test-bucket", pageNumber: 1, page: page}, + processingState{errorCount: &sync.Map{}, objectCount: &scanned, filteredCount: &filtered}, + sources.ChanReporter{Ch: make(chan *sources.Chunk, 1)}, + NewCheckpointer(ctx, &s.Progress, false), + ) + + _, reported := s.GetUnitProgressFor("test-bucket") + assert.False(t, reported) + assert.EqualValues(t, 1, s.GetProgress().SectionsRemaining) + assert.Equal(t, "Bucket: test-bucket", s.GetProgress().Message) +} + +func TestSource_SkipReason(t *testing.T) { + filter, err := newObjectFilter(nil, []string{"archive/"}, nil, nil) + require.NoError(t, err) + s := Source{maxObjectSize: 1024, objectFilter: filter} + + key, filteredKey, imageKey := "keep/obj.txt", "archive/obj.txt", "keep/photo.jpg" + size, oversize, empty := int64(512), int64(2048), int64(0) + + tests := []struct { + name string + obj s3types.Object + want string + }{ + {name: "standard object", obj: s3types.Object{Key: &key, Size: &size}, want: ""}, + {name: "at the size limit", obj: s3types.Object{Key: &key, Size: &s.maxObjectSize}, want: ""}, + {name: "over the size limit", obj: s3types.Object{Key: &key, Size: &oversize}, want: skipReasonSizeLimit}, + { + name: "glacier", + obj: s3types.Object{Key: &key, Size: &size, StorageClass: s3types.ObjectStorageClassGlacier}, + want: skipReasonStorageClass, + }, + { + name: "glacier instant retrieval", + obj: s3types.Object{Key: &key, Size: &size, StorageClass: s3types.ObjectStorageClassGlacierIr}, + want: skipReasonStorageClass, + }, + {name: "empty", obj: s3types.Object{Key: &key, Size: &empty}, want: skipReasonEmptyFile}, + {name: "incompatible extension", obj: s3types.Object{Key: &imageKey, Size: &size}, want: skipReasonIncompatibleExtension}, + {name: "filtered", obj: s3types.Object{Key: &filteredKey, Size: &size}, want: skipReasonObjectFilter}, + // The filter is checked first, matching the reason the skipped objects metric has always recorded. + {name: "filtered and over the size limit", obj: s3types.Object{Key: &filteredKey, Size: &oversize}, want: skipReasonObjectFilter}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, s.skipReason(tt.obj)) + }) + } } diff --git a/pkg/sources/s3/unit.go b/pkg/sources/s3/unit.go index 89cb6bb60d35..79a946b65717 100644 --- a/pkg/sources/s3/unit.go +++ b/pkg/sources/s3/unit.go @@ -2,12 +2,17 @@ package s3 import ( "fmt" + "strings" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) const SourceUnitKindBucket sources.SourceUnitKind = "bucket" +// unitIDSeparator joins the role and bucket halves of a unit ID. It appears in +// neither a role ARN nor an S3 bucket name, so splitting on it is unambiguous. +const unitIDSeparator = "|" + type S3SourceUnit struct { Bucket string Role string @@ -30,7 +35,26 @@ func (s S3SourceUnit) Display() string { func constructS3SourceUnitID(bucket string, role string) string { unitID := "" if role != "" { - unitID += role + "|" + unitID += role + unitIDSeparator } return unitID + bucket } + +// unitEnvelope is the JSON shape a host application persists for an +// enumerated unit between an enumerate pass and a later scan pass: the +// SourceUnit proto marshalled with encoding/json, where unit_data carries the +// original unit as base64-encoded bytes. +type unitEnvelope struct { + ID string `json:"id"` + Kind string `json:"kind,omitempty"` + Display string `json:"display,omitempty"` + UnitData string `json:"unit_data,omitempty"` +} + +// splitS3SourceUnitID reverses constructS3SourceUnitID. +func splitS3SourceUnitID(unitID string) (role, bucket string) { + if idx := strings.Index(unitID, unitIDSeparator); idx >= 0 { + return unitID[:idx], unitID[idx+1:] + } + return "", unitID +} diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.go index 562a93947762..c9aadf7e4b66 100644 --- a/pkg/sources/source_manager.go +++ b/pkg/sources/source_manager.go @@ -145,15 +145,19 @@ func (s *SourceManager) EnumerateAndScan(ctx context.Context, sourceName string, progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel)) if err := sem.Acquire(ctx, 1); err != nil { // Context cancelled. - progress.ReportError(Fatal{err}) - return progress.Ref(), Fatal{err} + fatalErr := Fatal{err} + progress.ReportError(fatalErr) + progress.finishWithCause(fatalErr) + return progress.Ref(), fatalErr } s.wg.Add(1) go func() { - // Call Finish after the semaphore has been released. + // Call Finish after the semaphore has been released, and release + // the WaitGroup only after Finish's hooks have run: Wait() closes + // the hooks, so it must not return while a hook can still fire. + defer s.wg.Done() defer progress.Finish() defer sem.Release(1) - defer s.wg.Done() ctx := context.WithValues(ctx, "source_manager_worker_id", common.RandomID(5), ) @@ -198,9 +202,11 @@ func (s *SourceManager) Enumerate(ctx context.Context, sourceName string, source s.wg.Add(1) go func() { - // Call Finish after the semaphore has been released. - defer progress.Finish() + // Release the WaitGroup only after Finish's hooks have run: Wait() + // closes the hooks, so it must not return while a hook can still + // fire. defer s.wg.Done() + defer progress.Finish() ctx := context.WithValues(ctx, "source_manager_worker_id", common.RandomID(5), ) @@ -235,15 +241,19 @@ func (s *SourceManager) Scan(ctx context.Context, sourceName string, source Sour progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel)) if err := s.sem.Acquire(ctx, 1); err != nil { // Context cancelled. - progress.ReportError(Fatal{err}) - return progress.Ref(), Fatal{err} + fatalErr := Fatal{err} + progress.ReportError(fatalErr) + progress.finishWithCause(fatalErr) + return progress.Ref(), fatalErr } s.wg.Add(1) go func() { - // Call Finish after the semaphore has been released. + // Call Finish after the semaphore has been released, and release + // the WaitGroup only after Finish's hooks have run: Wait() closes + // the hooks, so it must not return while a hook can still fire. + defer s.wg.Done() defer progress.Finish() defer s.sem.Release(1) - defer s.wg.Done() ctx := context.WithValues(ctx, "source_manager_worker_id", common.RandomID(5), ) @@ -408,7 +418,7 @@ func (s *SourceManager) enumerate(ctx context.Context, source Source, report *Jo ctx.Logger().Info("running source", "with_units", true) return s.enumerateWithUnits(ctx, enumChunker, report, reporter) } - return fmt.Errorf("Enumeration not supported or configured for source: %s", source.Type().String()) + return fmt.Errorf("enumeration not supported or configured for source: %s", source.Type().String()) } // scan runs a scan against a single SourceUnit as its only job. This method diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.go index 3cbfc025def4..d0c3575c6d92 100644 --- a/pkg/sources/source_manager_test.go +++ b/pkg/sources/source_manager_test.go @@ -1,10 +1,11 @@ package sources import ( - "errors" "fmt" "sort" + "sync" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -124,6 +125,7 @@ func TestSourceManagerRun(t *testing.T) { // The Chunks channel should be empty now. _, err = tryRead(mgr.Chunks()) assert.Error(t, err) + assert.ErrorIs(t, ref.Err(), ErrJobDone) } } @@ -361,8 +363,9 @@ func TestSourceManagerCancelRun(t *testing.T) { ref.CancelRun(cancelErr) <-ref.Done() assert.Error(t, ref.Snapshot().FatalError()) - assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr)) - assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr)) + assert.ErrorIs(t, ref.Snapshot().FatalError(), returnedErr) + assert.ErrorIs(t, ref.Snapshot().FatalErrors(), cancelErr) + assert.ErrorIs(t, ref.Err(), cancelErr) } func TestSourceManagerAvailableCapacity(t *testing.T) { @@ -511,3 +514,87 @@ func TestSourceManagerUnitHookNoUnits(t *testing.T) { assert.NotZero(t, m.ElapsedTime()) assert.Equal(t, 0, len(m.Errors)) } + +// TestUnitHookCloseConcurrentWithFinish tests that Close does not panic with +// "send on closed channel" when job goroutines are still ejecting finished +// metrics. Close must wait for in-flight sends and drop metrics that arrive +// after it. +func TestUnitHookCloseConcurrentWithFinish(t *testing.T) { + hook, ch := NewUnitHook(context.TODO()) + + // Drain the channel until Close closes it. + drained := make(chan struct{}) + go func() { + defer close(drained) + for range ch { + } + }() + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + ref := JobProgressRef{SourceID: SourceID(i), JobID: JobID(i)} + hook.StartUnitChunking(ref, nil, time.Now()) + hook.Finish(ref) + }(i) + } + assert.NoError(t, hook.Close()) + wg.Wait() + <-drained +} + +// TestSourceManagerSemaphoreAcquireError tests what happens when acquiring the +// semaphore tracking concurrent sources fails. +func TestSourceManagerSemaphoreAcquireError(t *testing.T) { + assertFinishedAfterAcquireError := func(t *testing.T, ref JobProgressRef, err error) { + t.Helper() + assert.IsType(t, Fatal{}, err) + + // The acquire should fail, so the job should be finished + select { + case <-ref.Done(): + default: + assert.FailNow(t, "job should have been finished") + } + + // Check that the ref has the same error as what the manager returns + assert.ErrorIs(t, ref.Err(), err) + assert.NotErrorIs(t, ref.Err(), ErrJobDone, "job should not be marked as done on failure to acquire semaphore") + } + + // Timeout for the context in these tests. + // The duration does not matter since synctest uses a fake clock. + const timeoutDuration = time.Hour + + t.Run("EnumerateAndScan", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + mgr := NewManager(WithConcurrentSources(0)) + source, err := buildDummy(&counterChunker{count: 1}) + assert.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + defer cancel() + ref, err := mgr.EnumerateAndScan(ctx, "dummy", source) + assertFinishedAfterAcquireError(t, ref, err) + // No job was started, so Wait should not block or error + assert.NoError(t, mgr.Wait()) + }) + }) + + t.Run("Scan", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + mgr := NewManager(WithConcurrentSources(0)) + source, err := buildDummy(&counterChunker{count: 1}) + assert.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + defer cancel() + ref, err := mgr.Scan(ctx, "dummy", source, countChunk(123)) + assertFinishedAfterAcquireError(t, ref, err) + // No job was started, so Wait should not block or error + assert.NoError(t, mgr.Wait()) + }) + }) +} diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index 88b11b85a5ea..c738c1ef88aa 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -309,9 +309,13 @@ type GithubConfig struct { Repos []string // Orgs is the list of organizations to scan. Orgs []string - // ExcludeRepos is a list of repositories to exclude from the scan. + // ExcludeRepos is a list of repositories to exclude from an org/user scan. + // It only filters repos enumerated via Orgs; it has no effect when Repos is set + // explicitly, since explicit repos bypass the filtering step entirely. ExcludeRepos []string - // IncludeRepos is a list of repositories to include in the scan. + // IncludeRepos is a list of repositories to include in an org/user scan. + // It only filters repos enumerated via Orgs; it has no effect when Repos is set + // explicitly, since explicit repos bypass the filtering step entirely. IncludeRepos []string // Filter is the filter to use to scan the source. Filter *common.Filter @@ -334,6 +338,8 @@ type GithubConfig struct { // NoCleanup allows to keeps cloned repositories in ClonePath after scanning instead of removing them. NoCleanup bool IgnoreGists bool + // ExcludeArchived indicates whether to exclude archived repositories from the scan. + ExcludeArchived bool // PrintLegacyJSON indicates whether to print legacy JSON output format for this source. PrintLegacyJSON bool } @@ -411,6 +417,18 @@ type S3Config struct { Roles []string // MaxObjectSize is the maximum object size to scan. MaxObjectSize int64 + // Endpoint targets an S3-compatible service other than AWS S3. + Endpoint string + // Region is the region used to sign requests. + Region string + // IncludePrefixes limits the scan to objects under these key prefixes. + IncludePrefixes []string + // ExcludePrefixes skips objects under these key prefixes. + ExcludePrefixes []string + // IncludeExtensions limits the scan to objects with these file extensions. + IncludeExtensions []string + // ExcludeExtensions skips objects with these file extensions. + ExcludeExtensions []string } // SyslogConfig defines the optional configuration for a syslog source. @@ -492,6 +510,8 @@ type Progress struct { EncodedResumeInfo string SectionsCompleted int32 SectionsRemaining int32 + // unitProgressByID is used for progress inside a unit (see below) + unitProgressByID map[string]UnitProgress } // Validator is an interface for validating a source. Sources can optionally implement this interface to validate @@ -617,3 +637,71 @@ func unmarshalEncodedResumeInfo(data string) map[string]string { } // -/sub-unit-resumption----------------------------------------------------------- + +// -unit-progress------------------------------------------------------------------ +// +// The following methods let a Source report how far it has got inside a unit, +// for units large enough that counting finished units reads as no progress. +// They are kept apart from EncodedResumeInfo, so reporting progress never +// changes where a resumed scan starts. +// +// Usage: +// - id should be the SourceUnit ID +// + +// UnitProgressState says whether the totals in a UnitProgress are final. +type UnitProgressState string + +const ( + // UnitProgressCounting means the source is still finding out how much the + // unit holds, so the totals are only a lower bound. + UnitProgressCounting UnitProgressState = "counting" + // UnitProgressReady means the totals are final. + UnitProgressReady UnitProgressState = "ready" + // UnitProgressUnavailable means the source could not find out how much the + // unit holds, so only the done counts mean anything. + UnitProgressUnavailable UnitProgressState = "unavailable" +) + +// UnitProgress is how far a Source has got through one unit. Items are +// whatever the source scans one at a time, such as objects or files. +type UnitProgress struct { + State UnitProgressState `json:"state"` + ItemsDone uint64 `json:"items_done"` + ItemsTotal uint64 `json:"items_total"` + BytesDone uint64 `json:"bytes_done"` + BytesTotal uint64 `json:"bytes_total"` + // ResumedBytes is the part of BytesDone that an earlier run of the unit + // finished, so a scan rate can leave it out. + ResumedBytes uint64 `json:"resumed_bytes"` +} + +// Percent returns the share of bytes done once the totals are final, and 0 +// before. It is capped at 99, because only the unit finishing makes it complete. +func (u UnitProgress) Percent() int64 { + if u.State != UnitProgressReady || u.BytesTotal == 0 { + return 0 + } + return int64(min(u.BytesDone*100/u.BytesTotal, 99)) +} + +// SetUnitProgressFor records the progress of the unit with the provided ID. +func (p *Progress) SetUnitProgressFor(id string, progress UnitProgress) { + p.mut.Lock() + defer p.mut.Unlock() + if p.unitProgressByID == nil { + p.unitProgressByID = make(map[string]UnitProgress) + } + p.unitProgressByID[id] = progress +} + +// GetUnitProgressFor returns the progress last recorded for the unit with the +// provided ID, and whether any was recorded. +func (p *Progress) GetUnitProgressFor(id string) (UnitProgress, bool) { + p.mut.Lock() + defer p.mut.Unlock() + progress, ok := p.unitProgressByID[id] + return progress, ok +} + +// -/unit-progress----------------------------------------------------------------- diff --git a/pkg/sources/sources_test.go b/pkg/sources/sources_test.go index 6db3fccbc79b..fc6fd22b4236 100644 --- a/pkg/sources/sources_test.go +++ b/pkg/sources/sources_test.go @@ -14,3 +14,65 @@ func TestChunkSize(t *testing.T) { t.Parallel() assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(104), "Chunk struct size exceeds 104 bytes") } + +func TestProgress_UnitProgressKeepsResumeInfo(t *testing.T) { + t.Parallel() + var p Progress + p.SetEncodedResumeInfoFor("unit", "key") + resumeInfo := p.EncodedResumeInfo + + want := UnitProgress{State: UnitProgressReady, ItemsDone: 1, ItemsTotal: 2, BytesDone: 10, BytesTotal: 20} + p.SetUnitProgressFor("unit", want) + + got, ok := p.GetUnitProgressFor("unit") + assert.True(t, ok) + assert.Equal(t, want, got) + assert.Equal(t, resumeInfo, p.EncodedResumeInfo) + assert.Equal(t, "key", p.GetEncodedResumeInfoFor("unit")) + + _, ok = p.GetUnitProgressFor("other unit") + assert.False(t, ok) +} + +func TestUnitProgress_Percent(t *testing.T) { + t.Parallel() + tests := []struct { + name string + progress UnitProgress + want int64 + }{ + {name: "nothing reported", progress: UnitProgress{}, want: 0}, + { + name: "still counting", + progress: UnitProgress{State: UnitProgressCounting, BytesDone: 500, BytesTotal: 1000}, + want: 0, + }, + { + name: "count unavailable", + progress: UnitProgress{State: UnitProgressUnavailable, BytesDone: 500, BytesTotal: 1000}, + want: 0, + }, + {name: "empty total", progress: UnitProgress{State: UnitProgressReady}, want: 0}, + { + name: "partial rounds down", + progress: UnitProgress{State: UnitProgressReady, BytesDone: 1999, BytesTotal: 3000}, + want: 66, + }, + { + name: "all done caps at 99", + progress: UnitProgress{State: UnitProgressReady, BytesDone: 1000, BytesTotal: 1000}, + want: 99, + }, + { + name: "done past total caps at 99", + progress: UnitProgress{State: UnitProgressReady, BytesDone: 1200, BytesTotal: 1000}, + want: 99, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.progress.Percent()) + }) + } +} diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.go index dc1a1af6c446..6e8bf1905eb9 100644 --- a/pkg/sources/syslog/syslog.go +++ b/pkg/sources/syslog/syslog.go @@ -83,13 +83,13 @@ func (s *Source) Validate(ctx context.Context) []error { if err != nil { errs = append(errs, fmt.Errorf("error listening on tcp socket: %s", err)) } - srv.Close() + _ = srv.Close() case "udp": srv, err := net.ListenPacket(s.conn.Protocol, s.conn.ListenAddress) if err != nil { errs = append(errs, fmt.Errorf("error listening on udp socket: %s", err)) } - srv.Close() + _ = srv.Close() } } if s.conn.Protocol != "tcp" && s.conn.Protocol != "udp" { @@ -198,7 +198,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . if err != nil { return errors.WrapPrefix(err, "error creating TLS listener", 0) } - defer lis.Close() + defer func() { _ = lis.Close() }() return s.acceptTCPConnections(ctx, lis, chunksChan) case s.conn.Protocol == "tcp": @@ -206,7 +206,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . if err != nil { return errors.WrapPrefix(err, "error creating TCP listener", 0) } - defer lis.Close() + defer func() { _ = lis.Close() }() return s.acceptTCPConnections(ctx, lis, chunksChan) case s.conn.Protocol == "udp": @@ -218,7 +218,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . if err != nil { return errors.WrapPrefix(err, "could not set UDP deadline", 0) } - defer lis.Close() + defer func() { _ = lis.Close() }() return s.acceptUDPConnections(ctx, lis, chunksChan) default: diff --git a/pkg/sources/test_helpers.go b/pkg/sources/test_helpers.go index 25f59af8ae09..965f46ae6065 100644 --- a/pkg/sources/test_helpers.go +++ b/pkg/sources/test_helpers.go @@ -8,7 +8,8 @@ import ( type ChunkFunc func(chunk *Chunk) error -var MatchError = errors.New("chunk doesn't match") +// ErrMatch indicates a chunk did not match and the helper should keep waiting. +var ErrMatch = errors.New("chunk doesn't match") func HandleTestChannel(chunksCh chan *Chunk, cf ChunkFunc) error { for { @@ -16,7 +17,7 @@ func HandleTestChannel(chunksCh chan *Chunk, cf ChunkFunc) error { case gotChunk := <-chunksCh: err := cf(gotChunk) if err != nil { - if errors.Is(err, MatchError) { + if errors.Is(err, ErrMatch) { continue } return err diff --git a/pkg/ssrf/ssrf.go b/pkg/ssrf/ssrf.go new file mode 100644 index 000000000000..763f074a9eef --- /dev/null +++ b/pkg/ssrf/ssrf.go @@ -0,0 +1,177 @@ +// Package ssrf guards outbound connections whose destinations are taken from +// scanned or otherwise untrusted content (a secret's "domain"/"endpoint", +// a connection string, a URL in a config file). Without a guard, an attacker +// who can plant such content can steer an HTTP client at internal-only +// addresses, and redirects turn an attacker-owned endpoint into a pivot into +// internal space (cluster services, the cloud metadata server, etc.). +// +// The check runs inside the dialer (via ControlContext), after DNS +// resolution and immediately before the socket connects. That placement +// matters for two reasons: +// - It is DNS-rebinding safe: a hostname that resolves to a public IP on a +// pre-flight check but to 169.254.169.254 at connect time is still caught, +// because the check runs on the address actually being dialed. +// - It covers HTTP redirects: any redirect hop that opens a new connection +// re-dials through the guarded dialer, and every fresh dial re-checks the +// resolved IP. +// +// Caveat: the guard inspects the address the transport dials. If an HTTP +// forward proxy is configured (HTTP(S)_PROXY), the transport dials the proxy +// and the proxy makes the final connection, so egress policy must also be +// enforced at the proxy. +package ssrf + +import ( + "context" + "errors" + "fmt" + "net" + "syscall" +) + +// ErrEgressBlocked is wrapped by every error this package returns when it +// refuses a connection. Callers can errors.Is against it to treat a blocked +// egress as an expected, benign outcome rather than a failure. +var ErrEgressBlocked = errors.New("egress blocked: non-public address") + +// GuardDialer returns a copy of base (or a zero dialer when base is nil) +// that refuses non-public targets via CheckDialAddress. The check is +// installed as ControlContext because the net package ignores Control +// whenever ControlContext is set: installing the guard as Control would let +// a base dialer carrying a ControlContext bypass it entirely. Any +// ControlContext or Control already set on base runs after the check, for +// allowed targets only. +func GuardDialer(base *net.Dialer) *net.Dialer { + d := net.Dialer{} + if base != nil { + d = *base + } + prevCtx, prevCtl := d.ControlContext, d.Control + d.Control = nil + d.ControlContext = func(ctx context.Context, network, address string, c syscall.RawConn) error { + if err := CheckDialAddress(address); err != nil { + return err + } + if prevCtx != nil { + return prevCtx(ctx, network, address, c) + } + if prevCtl != nil { + return prevCtl(network, address, c) + } + return nil + } + return &d +} + +// CheckDialAddress rejects a dial target whose IP is not a public address. +// Every rejection wraps ErrEgressBlocked so callers can recognize it. The +// address must be a resolved "ip:port" pair as seen by a dialer Control hook; +// anything else is refused (fail closed). +func CheckDialAddress(address string) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + // Fail closed: if we cannot understand the target, do not connect. + // %v (not %w) on the parse error: a second %w would make this a + // multi-branch error tree, and whole-tree checks like thog's + // IsEgressBlockedOnly would classify the rejection as a real failure. + return fmt.Errorf("ssrf guard: cannot parse dial address %q: %v: %w", address, err, ErrEgressBlocked) + } + ip := net.ParseIP(host) + if ip == nil { + // At dial time the address is already resolved to an IP literal. If it + // is not, refuse rather than risk connecting to something unvetted. + return fmt.Errorf("ssrf guard: dial target %q is not an IP literal: %w", host, ErrEgressBlocked) + } + if IsNonPublicIP(ip) { + return fmt.Errorf("ssrf guard: refusing to connect to non-public address %s: %w", ip, ErrEgressBlocked) + } + return nil +} + +// extraBlockedCIDRs covers ranges not classified by the net.IP helper methods +// but that must never be reachable from guarded egress. +var extraBlockedCIDRs = mustParseCIDRs( + // IPv4 special-use / non-public ranges. + "0.0.0.0/8", // RFC1122 "this network"; parts route to localhost on some stacks + "100.64.0.0/10", // RFC6598 CGNAT (incl. Alibaba metadata 100.100.100.200) + "192.0.0.0/24", // RFC6890 IETF protocol assignments + "192.0.2.0/24", // TEST-NET-1 + "198.18.0.0/15", // RFC2544 benchmarking + "198.51.100.0/24", // TEST-NET-2 + "203.0.113.0/24", // TEST-NET-3 + "240.0.0.0/4", // RFC1112 class E reserved (also covers 255.255.255.255 broadcast) + // IPv6 embeddings of an IPv4 address that To4() does NOT normalize; without + // these, an internal v4 target can be smuggled in as an IPv6 literal. The + // NAT64 well-known prefix is NOT here: DNS64 legitimately synthesizes it for + // public IPv4-only endpoints, so IsNonPublicIP extracts and classifies the + // embedded v4 instead (see nat64WellKnownPrefix). The mechanisms below are + // deprecated or operator-local, so there is no availability reason to allow + // any of them and they are blocked outright. + "::/96", // RFC4291 IPv4-compatible IPv6, deprecated (:: and ::1 are caught earlier) + "::ffff:0:0:0/96", // RFC2765 SIIT "IPv4-translated"; To4() only normalizes ::ffff:0:0/96 + "64:ff9b:1::/48", // RFC8215 NAT64 local-use prefix; embedded position varies per operator + "2002::/16", // RFC3056 6to4, deprecated + "2001::/32", // RFC4380 Teredo; embeds v4 server/client addresses + // IPv6 ranges with internal or non-routable scope. + "fec0::/10", // RFC3879 site-local, deprecated but still routed as internal scope by legacy gear + // IPv6 special-use parity with the v4 test/doc ranges above. + "2001:db8::/32", // RFC3849 documentation + "100::/64", // RFC6666 discard-only + // Note: IPv4-mapped IPv6 (e.g. ::ffff:169.254.169.254) is handled by the + // To4() normalization in IsNonPublicIP, not by a CIDR here, because a + // ::ffff:0:0/96 entry would match every IPv4 address. +) + +// nat64WellKnownPrefix is the RFC6052 NAT64 well-known prefix 64:ff9b::/96. +// Unlike the deprecated embeddings in extraBlockedCIDRs, DNS64 resolvers +// synthesize these addresses for ordinary public IPv4-only endpoints, so +// blanket-blocking the prefix would break every guarded dial to an IPv4-only +// host in an IPv6-only (DNS64/NAT64) network. Instead the embedded IPv4 in +// the low 32 bits is extracted and classified on its own. +var nat64WellKnownPrefix = mustParseCIDRs("64:ff9b::/96")[0] + +// IsNonPublicIP reports whether an IP must not be dialed by a guarded client: +// loopback, link-local (incl. cloud metadata), private (RFC1918/RFC4193), +// CGNAT, multicast, unspecified, the special-use ranges above, and IPv6 +// embeddings of any of those. +func IsNonPublicIP(ip net.IP) bool { + if ip == nil { + return true // fail closed + } + // Normalize IPv4-in-IPv6 so the v4 classification methods apply. + if v4 := ip.To4(); v4 != nil { + ip = v4 + } else if ip16 := ip.To16(); ip16 != nil && nat64WellKnownPrefix.Contains(ip16) { + // NAT64 well-known prefix: classify the embedded IPv4 (low 32 bits) so + // DNS64-synthesized addresses of public endpoints stay reachable while + // embeddings of internal targets are still blocked. + ip = net.IPv4(ip16[12], ip16[13], ip16[14], ip16[15]).To4() + } + + if ip.IsLoopback() || // 127.0.0.0/8, ::1 + ip.IsUnspecified() || // 0.0.0.0, :: + ip.IsPrivate() || // RFC1918 (10/8, 172.16/12, 192.168/16) + RFC4193 (fc00::/7) + ip.IsLinkLocalUnicast() || // 169.254.0.0/16 (metadata 169.254.169.254, ECS creds 169.254.170.2), fe80::/10 + ip.IsMulticast() { // 224.0.0.0/4, ff00::/8 (subsumes link-local and interface-local multicast) + return true + } + + for _, n := range extraBlockedCIDRs { + if n.Contains(ip) { + return true + } + } + return false +} + +func mustParseCIDRs(cidrs ...string) []*net.IPNet { + nets := make([]*net.IPNet, 0, len(cidrs)) + for _, c := range cidrs { + _, n, err := net.ParseCIDR(c) + if err != nil { + panic(fmt.Sprintf("ssrf guard: invalid CIDR %q: %v", c, err)) + } + nets = append(nets, n) + } + return nets +} diff --git a/pkg/ssrf/ssrf_test.go b/pkg/ssrf/ssrf_test.go new file mode 100644 index 000000000000..c059d6fb84bb --- /dev/null +++ b/pkg/ssrf/ssrf_test.go @@ -0,0 +1,190 @@ +package ssrf + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsNonPublicIP(t *testing.T) { + cases := []struct { + ip string + blocked bool + }{ + // Public: allowed. + {"8.8.8.8", false}, + {"1.1.1.1", false}, + {"93.184.216.34", false}, // example.com + {"2606:2800:220:1:248:1893:25c8:1946", false}, + + // Loopback. + {"127.0.0.1", true}, + {"127.1.2.3", true}, + {"::1", true}, + + // Link-local incl. cloud metadata and ECS credentials endpoint. + {"169.254.169.254", true}, // GCP/AWS/Azure metadata + {"169.254.170.2", true}, // AWS ECS task role credentials + {"fe80::1", true}, + + // Private RFC1918 / RFC4193. + {"10.0.0.5", true}, + {"172.16.31.9", true}, + {"192.168.1.1", true}, + {"fc00::1", true}, + {"fd12:3456::1", true}, + + // CGNAT (incl. Alibaba metadata 100.100.100.200). + {"100.64.0.1", true}, + {"100.100.100.200", true}, + + // Unspecified + "this network" 0.0.0.0/8. + {"0.0.0.0", true}, + {"0.1.2.3", true}, + {"::", true}, + + // Class E reserved + broadcast. + {"240.0.0.1", true}, + {"255.255.255.255", true}, + + // Test-net / protocol-assignment ranges. + {"192.0.2.5", true}, + {"198.18.0.1", true}, + {"203.0.113.9", true}, + + // IPv4-mapped IPv6 must not bypass the v4 checks. + {"::ffff:169.254.169.254", true}, + {"::ffff:10.0.0.1", true}, + {"::ffff:8.8.8.8", false}, + + // IPv6 embeddings of an internal v4 target (To4() does not normalize these). + {"64:ff9b::a00:1", true}, // NAT64 of 10.0.0.1 + {"2002:0a00:0001::1", true}, // 6to4 of 10.0.0.1 + {"::0a00:0001", true}, // IPv4-compatible ::10.0.0.1 + {"::ffff:0:a00:1", true}, // SIIT IPv4-translated ::ffff:0:10.0.0.1 + {"2001::a00:1", true}, // Teredo (embeds v4 addresses) + {"2001:db8::1", true}, // documentation + + // NAT64 well-known prefix classifies the EMBEDDED v4: DNS64-synthesized + // addresses of public IPv4-only endpoints must stay reachable in + // IPv6-only networks, while internal embeddings stay blocked. + {"64:ff9b::808:808", false}, // NAT64 of 8.8.8.8 + {"64:ff9b::a9fe:a9fe", true}, // NAT64 of 169.254.169.254 + {"64:ff9b:1::a00:1", true}, // RFC8215 local-use prefix stays blanket-blocked + + // Deprecated IPv6 site-local, still internal scope on legacy gear. + {"fec0::1", true}, + {"feff::1", true}, + } + + for _, c := range cases { + ip := net.ParseIP(c.ip) + require.NotNilf(t, ip, "bad test IP %q", c.ip) + assert.Equalf(t, c.blocked, IsNonPublicIP(ip), "IsNonPublicIP(%s)", c.ip) + } + + assert.True(t, IsNonPublicIP(nil), "nil IP must fail closed") +} + +func TestCheckDialAddress(t *testing.T) { + cases := []struct { + name string + address string + blocked bool + }{ + {"public IP allowed", "8.8.8.8:443", false}, + {"loopback blocked", "127.0.0.1:8080", true}, + {"metadata blocked", "169.254.169.254:80", true}, + {"IPv6 loopback blocked", "[::1]:443", true}, + {"unparseable address blocked", "no-port-here", true}, + {"unresolved hostname blocked", "example.com:443", true}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := CheckDialAddress(c.address) + if !c.blocked { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrEgressBlocked) + assert.Contains(t, err.Error(), "ssrf guard") + }) + } +} + +// TestGuardDialer_BlocksLoopback drives a real dial through the guarded dialer +// at a loopback httptest server and verifies the refusal carries the sentinel. +func TestGuardDialer_BlocksLoopback(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + dialer := GuardDialer(nil) + _, err := dialer.DialContext(context.Background(), "tcp", strings.TrimPrefix(srv.URL, "http://")) + require.Error(t, err) + assert.ErrorIs(t, err, ErrEgressBlocked) +} + +// TestGuardDialer_ChainsControl verifies an existing Control hook on the base +// dialer still runs (after the guard check) for allowed targets. +func TestGuardDialer_ChainsControl(t *testing.T) { + var chained bool + base := &net.Dialer{ + Control: func(_, _ string, _ syscall.RawConn) error { + chained = true + return nil + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + // Blocked target: the guard refuses before the chained Control runs. + dialer := GuardDialer(base) + _, err := dialer.DialContext(ctx, "tcp", "127.0.0.1:1") + require.ErrorIs(t, err, ErrEgressBlocked) + assert.False(t, chained, "chained Control must not run for blocked targets") + + // Allowed target: the chained Control runs. The dial itself may fail + // (network-dependent); only the Control invocation matters here. + _, _ = dialer.DialContext(ctx, "tcp", "1.1.1.1:53") + assert.True(t, chained, "chained Control must run for allowed targets") +} + +// TestGuardDialer_GuardsBaseWithControlContext locks in the fix for a bypass: +// the net package ignores Control whenever ControlContext is set, so a guard +// installed as Control would never run for a base dialer carrying a +// ControlContext. The guard must win regardless of which hook the base uses. +func TestGuardDialer_GuardsBaseWithControlContext(t *testing.T) { + var chained bool + base := &net.Dialer{ + ControlContext: func(_ context.Context, _, _ string, _ syscall.RawConn) error { + chained = true + return nil + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + dialer := GuardDialer(base) + _, err := dialer.DialContext(ctx, "tcp", "127.0.0.1:1") + require.ErrorIs(t, err, ErrEgressBlocked, + "a base ControlContext must not bypass the guard") + assert.False(t, chained, "chained ControlContext must not run for blocked targets") + + // Allowed target: the base's ControlContext still runs after the check. + _, _ = dialer.DialContext(ctx, "tcp", "1.1.1.1:53") + assert.True(t, chained, "chained ControlContext must run for allowed targets") +} diff --git a/pkg/tui/common/style.go b/pkg/tui/common/style.go index 2003869add3e..3260eebbec69 100644 --- a/pkg/tui/common/style.go +++ b/pkg/tui/common/style.go @@ -18,7 +18,7 @@ func StyleConfig() gansi.StyleConfig { s.H1.Prefix = "# " s.H1.Suffix = "" s.H1.Color = strptr("39") - s.Document.StylePrimitive.Color = noColor + s.Document.Color = noColor s.CodeBlock.Chroma.Text.Color = noColor s.CodeBlock.Chroma.Name.Color = noColor // This fixes an issue with the default style config. For example diff --git a/pkg/tui/components/selector/selector.go b/pkg/tui/components/selector/selector.go index 3b1b2ebb08e0..d1c908c978d2 100644 --- a/pkg/tui/components/selector/selector.go +++ b/pkg/tui/components/selector/selector.go @@ -50,22 +50,22 @@ func New(common common.Common, items []IdentifiableItem, delegate ItemDelegate) // PerPage returns the number of items per page. func (s *Selector) PerPage() int { - return s.Model.Paginator.PerPage + return s.Paginator.PerPage } // SetPage sets the current page. func (s *Selector) SetPage(page int) { - s.Model.Paginator.Page = page + s.Paginator.Page = page } // Page returns the current page. func (s *Selector) Page() int { - return s.Model.Paginator.Page + return s.Paginator.Page } // TotalPages returns the total number of pages. func (s *Selector) TotalPages() int { - return s.Model.Paginator.TotalPages + return s.Paginator.TotalPages } // Select selects the item at the given index. @@ -138,7 +138,7 @@ func (s *Selector) Update(msg tea.Msg) (tea.Model, tea.Cmd) { cmds := make([]tea.Cmd, 0) switch msg := msg.(type) { case tea.KeyMsg: - filterState := s.Model.FilterState() + filterState := s.FilterState() switch { case key.Matches(msg, s.common.KeyMap.Help): if filterState == list.Filtering { @@ -158,7 +158,7 @@ func (s *Selector) Update(msg tea.Msg) (tea.Model, tea.Cmd) { cmds = append(cmds, cmd) } // Track filter state and update active item when filter state changes. - filterState := s.Model.FilterState() + filterState := s.FilterState() if s.filterState != filterState { cmds = append(cmds, s.activeFilterCmd) } @@ -182,7 +182,7 @@ func (s *Selector) SelectItem() tea.Msg { } func (s *Selector) selectCmd() tea.Msg { - item := s.Model.SelectedItem() + item := s.SelectedItem() i, ok := item.(IdentifiableItem) if !ok { return SelectMsg{} @@ -191,7 +191,7 @@ func (s *Selector) selectCmd() tea.Msg { } func (s *Selector) activeCmd() tea.Msg { - item := s.Model.SelectedItem() + item := s.SelectedItem() i, ok := item.(IdentifiableItem) if !ok { return ActiveMsg{} @@ -203,7 +203,7 @@ func (s *Selector) activeFilterCmd() tea.Msg { // Here we use VisibleItems because when list.FilterMatchesMsg is sent, // VisibleItems is the only way to get the list of filtered items. The list // bubble should export something like list.FilterMatchesMsg.Items(). - items := s.Model.VisibleItems() + items := s.VisibleItems() if len(items) == 0 { return nil } diff --git a/pkg/tui/components/textinputs/textinputs.go b/pkg/tui/components/textinputs/textinputs.go index 5f5a41804155..e90e2aa2d23a 100644 --- a/pkg/tui/components/textinputs/textinputs.go +++ b/pkg/tui/components/textinputs/textinputs.go @@ -18,11 +18,17 @@ var ( helpStyle = blurredStyle // cursorStyle = focusedStyle. // cursorModeHelpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("244")) - - focusedSkipButton = lipgloss.NewStyle().Foreground(lipgloss.Color("205")).Render("[ Run with defaults ]") - blurredSkipButton = fmt.Sprintf("[ %s ]", lipgloss.NewStyle().Foreground(lipgloss.Color("240")).Render("Run with defaults")) ) +func skipButtonView(focused bool) string { + // Keep rendering lazy. Rendering lipgloss styles at package init can make + // terminal color detection write escape sequences before non-TUI commands run. + if focused { + return focusedStyle.Render("[ Run with defaults ]") + } + return fmt.Sprintf("[ %s ]", blurredStyle.Render("Run with defaults")) +} + // SelectNextMsg used for emitting events when the 'Next' button is selected. type SelectNextMsg int @@ -179,11 +185,7 @@ func (m Model) View() string { } if m.skipButton { - button := &blurredSkipButton - if m.focusIndex == -1 { - button = &focusedSkipButton - } - fmt.Fprintf(&b, "%s\n\n\n", *button) + fmt.Fprintf(&b, "%s\n\n\n", skipButtonView(m.focusIndex == -1)) } for i := range m.inputs { diff --git a/pkg/tui/components/textinputs/textinputs_test.go b/pkg/tui/components/textinputs/textinputs_test.go new file mode 100644 index 000000000000..213e9efd789e --- /dev/null +++ b/pkg/tui/components/textinputs/textinputs_test.go @@ -0,0 +1,26 @@ +package textinputs + +import ( + "strings" + "testing" +) + +func TestSkipButtonViewIsRenderedLazily(t *testing.T) { + m := New(nil).SetSkip(true) + + view := m.View() + + if !strings.Contains(view, "Run with defaults") { + t.Fatalf("expected skip button text in view, got %q", view) + } +} + +func TestSkipButtonViewIsHiddenByDefault(t *testing.T) { + m := New(nil) + + view := m.View() + + if strings.Contains(view, "Run with defaults") { + t.Fatalf("did not expect skip button text in default view, got %q", view) + } +} diff --git a/pkg/tui/components/viewport/viewport.go b/pkg/tui/components/viewport/viewport.go index 77fa618285b7..df95cb10522f 100644 --- a/pkg/tui/components/viewport/viewport.go +++ b/pkg/tui/components/viewport/viewport.go @@ -25,8 +25,8 @@ func New(c common.Common) *Viewport { // SetSize implements common.Component. func (v *Viewport) SetSize(width, height int) { v.common.SetSize(width, height) - v.Model.Width = width - v.Model.Height = height + v.Width = width + v.Height = height } // Init implements tea.Model. diff --git a/pkg/tui/pages/analyze_form/analyze_form.go b/pkg/tui/pages/analyze_form/analyze_form.go index 6448a3f974b7..177db9c93468 100644 --- a/pkg/tui/pages/analyze_form/analyze_form.go +++ b/pkg/tui/pages/analyze_form/analyze_form.go @@ -188,7 +188,7 @@ func (ui *AnalyzeForm) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return ui, nil case tea.KeyMsg: switch { - case key.Matches(msg, ui.Common.KeyMap.Back): + case key.Matches(msg, ui.KeyMap.Back): return nil, tea.Quit } } diff --git a/pkg/tui/pages/analyze_keys/analyze_keys.go b/pkg/tui/pages/analyze_keys/analyze_keys.go index d85bc6e85602..082a630f711e 100644 --- a/pkg/tui/pages/analyze_keys/analyze_keys.go +++ b/pkg/tui/pages/analyze_keys/analyze_keys.go @@ -62,9 +62,9 @@ func (ui *AnalyzeKeyPage) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ui.list.SetSize(msg.Width-h, msg.Height-v) case tea.KeyMsg: switch { - case key.Matches(msg, ui.Common.KeyMap.Back): + case key.Matches(msg, ui.KeyMap.Back): return nil, tea.Quit - case key.Matches(msg, ui.Common.KeyMap.Select): + case key.Matches(msg, ui.KeyMap.Select): chosenAnalyzer := ui.list.SelectedItem().(KeyTypeItem) return ui, func() tea.Msg { diff --git a/pkg/tui/pages/wizard_intro/item.go b/pkg/tui/pages/wizard_intro/item.go index 7fbefa704dec..5ae19e2c4fc6 100644 --- a/pkg/tui/pages/wizard_intro/item.go +++ b/pkg/tui/pages/wizard_intro/item.go @@ -129,7 +129,7 @@ func (d ItemDelegate) Render(w io.Writer, m list.Model, index int, listItem list cmd = styles.Command.Render(cmd) s.WriteString(cmd) - fmt.Fprint(w, + _, _ = fmt.Fprint(w, d.common.Zone.Mark(i.ID(), styles.Base.Render(s.String()), ), diff --git a/pkg/tui/pages/wizard_intro/wizard_intro.go b/pkg/tui/pages/wizard_intro/wizard_intro.go index 4e54e9dfea1e..861e132b8484 100644 --- a/pkg/tui/pages/wizard_intro/wizard_intro.go +++ b/pkg/tui/pages/wizard_intro/wizard_intro.go @@ -101,8 +101,8 @@ func (m *WizardIntro) View() string { func (m *WizardIntro) ShortHelp() []key.Binding { kb := make([]key.Binding, 0) kb = append(kb, - m.Common.KeyMap.UpDown, - m.Common.KeyMap.Section, + m.KeyMap.UpDown, + m.KeyMap.Section, ) return kb } diff --git a/pkg/updater/updater.go b/pkg/updater/updater.go index b8ad42020dbe..a1b06a440c0b 100644 --- a/pkg/updater/updater.go +++ b/pkg/updater/updater.go @@ -75,7 +75,7 @@ func (g *OSS) Fetch() (io.Reader, error) { if err != nil || resp == nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNoContent { return nil, errors.New("already up to date") @@ -105,7 +105,7 @@ func (g *OSS) Fetch() (io.Reader, error) { if err != nil { return nil, errors.Errorf("Failed to read gzip archive: %s", err) } - defer gzipReader.Close() + defer func() { _ = gzipReader.Close() }() tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() diff --git a/pkg/verificationcache/metrics_reporter.go b/pkg/verificationcache/metrics_reporter.go index c5f16865c3e9..3a9974266522 100644 --- a/pkg/verificationcache/metrics_reporter.go +++ b/pkg/verificationcache/metrics_reporter.go @@ -1,6 +1,10 @@ package verificationcache -import "time" +import ( + "time" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) // MetricsReporter is an interface used by a verification cache to report various metrics related to its operation. // Implementations must be thread-safe. @@ -10,7 +14,9 @@ type MetricsReporter interface { // smaller than the cache hit count due to cache hit "wasting"; see AddResultCacheHitsWasted for more information. AddCredentialVerificationsSaved(count int) - // AddFromDataVerifyTimeSpent records wall time spent in calls to detector.FromData with verify=true. + // AddFromDataVerifyTimeSpent records wall time spent verifying credentials remotely, either in a call to + // detector.FromData with verify=true or, for detectors that implement detectors.ResultVerifier, in the + // per-result verification of cache misses. AddFromDataVerifyTimeSpent(wallTime time.Duration) // AddResultCacheHits records result cache hits. Not all cache hits result in elided remote verification requests @@ -26,3 +32,19 @@ type MetricsReporter interface { // doesn't save any remote requests. AddResultCacheHitsWasted(count int) } + +// DetectorMetricsReporter is an optional interface that a MetricsReporter can additionally implement to receive +// verification timing attributed to the detector that incurred it. Implementations must be thread-safe. +// +// What one sample covers depends on the path taken. When the result cache is consulted, a detectors.ResultVerifier +// records one sample per remote VerifyResult call on a cache miss. Every other case records one sample per whole +// FromData(verify=true) pass, including its regex pass: detectors that do not implement detectors.ResultVerifier, +// targeted rescans (forceCacheUpdate), and a VerificationCache with no result cache. The last two apply to +// ResultVerifier detectors too, so one such detector can produce whole-pass samples that cover many remote calls +// alongside its per-call samples. A detector that starts implementing detectors.ResultVerifier shifts its +// distribution, which resets its baseline rather than signaling a provider change. +type DetectorMetricsReporter interface { + // AddDetectorVerifyTimeSpent records wall time spent verifying credentials remotely for one detector. It is not + // called for results served from the cache. + AddDetectorVerifyTimeSpent(detectorType detector_typepb.DetectorType, wallTime time.Duration) +} diff --git a/pkg/verificationcache/verification_cache.go b/pkg/verificationcache/verification_cache.go index 97ea3f93af70..83d9528bd8f2 100644 --- a/pkg/verificationcache/verification_cache.go +++ b/pkg/verificationcache/verification_cache.go @@ -9,13 +9,15 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/hasher" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) // VerificationCache is a structure that can be used to cache verification results from detectors so that a given // credential does not trigger multiple identical remote verification attempts. type VerificationCache struct { - metrics MetricsReporter - resultCache ResultCache + metrics MetricsReporter + resultCache ResultCache + detectorMetrics DetectorMetricsReporter // optional hashMu sync.Mutex hasher hasher.Hasher @@ -28,11 +30,31 @@ func New(resultCache ResultCache, metrics MetricsReporter) *VerificationCache { if metrics == nil { metrics = &InMemoryMetrics{} } + detectorMetrics, _ := metrics.(DetectorMetricsReporter) return &VerificationCache{ - metrics: metrics, - resultCache: resultCache, - hasher: hasher.NewBlake2B(), + metrics: metrics, + resultCache: resultCache, + detectorMetrics: detectorMetrics, + hasher: hasher.NewBlake2B(), + } +} + +// recordVerifyTime reports one remote verification pass to both metric views: the cross-detector aggregate and, when +// the reporter opted in, the per-detector histogram. Used by the paths where a single detector.FromData call with +// verify=true is the whole verification pass. +func (v *VerificationCache) recordVerifyTime(detectorType detector_typepb.DetectorType, wallTime time.Duration) { + v.metrics.AddFromDataVerifyTimeSpent(wallTime) + v.recordDetectorVerifyTime(detectorType, wallTime) +} + +// recordDetectorVerifyTime reports one verification sample to the per-detector view. It is a no-op for reporters that +// do not implement DetectorMetricsReporter, which keeps existing MetricsReporter implementations unaffected. +// verifyCacheMisses calls it directly because it samples each VerifyResult call per detector but reports the aggregate +// once per chunk. +func (v *VerificationCache) recordDetectorVerifyTime(detectorType detector_typepb.DetectorType, wallTime time.Duration) { + if v.detectorMetrics != nil { + v.detectorMetrics.AddDetectorVerifyTimeSpent(detectorType, wallTime) } } @@ -47,6 +69,9 @@ func New(resultCache ResultCache, metrics MetricsReporter) *VerificationCache { // returned result. If there is a cache hit for each result, these cached values are all returned. Otherwise, the // detector's FromData method is called again, but with verify=true, and the results are stored in the cache and then // returned. +// +// Detectors that implement detectors.ResultVerifier are handled by verifyCacheMisses instead, which verifies only the +// results that missed the cache rather than re-running the detector over the whole chunk. func (v *VerificationCache) FromData( ctx context.Context, detector detectors.Detector, @@ -59,7 +84,7 @@ func (v *VerificationCache) FromData( if verify { start := time.Now() defer func() { - v.metrics.AddFromDataVerifyTimeSpent(time.Since(start)) + v.recordVerifyTime(detector.Type(), time.Since(start)) }() } @@ -76,6 +101,11 @@ func (v *VerificationCache) FromData( return withoutRemoteVerification, nil } + // avoiding re-running verification for every result in the chunk. + // only if a detector implements detectors.ResultVerifier + if resultVerifier, ok := detector.(detectors.ResultVerifier); ok { + return v.verifyCacheMisses(ctx, resultVerifier, detector.Type(), withoutRemoteVerification) + } isEverythingCached := true var cacheHitsInCurrentChunk int for i, r := range withoutRemoteVerification { @@ -109,7 +139,7 @@ func (v *VerificationCache) FromData( start := time.Now() withRemoteVerification, err := detector.FromData(ctx, verify, data) if verify { - v.metrics.AddFromDataVerifyTimeSpent(time.Since(start)) + v.recordVerifyTime(detector.Type(), time.Since(start)) } if err != nil { return nil, err @@ -125,14 +155,66 @@ func (v *VerificationCache) FromData( copyForCaching := r // Do not persist raw secret values in a long-lived cache - copyForCaching.Raw = nil - copyForCaching.RawV2 = nil + copyForCaching.ClearSecrets() v.resultCache.Set(string(cacheKey), copyForCaching) } return withRemoteVerification, nil } +// verifyCacheMisses serves the results whose verification status is already cached and +// remotely verifies only the misses. +func (v *VerificationCache) verifyCacheMisses( + ctx context.Context, + detector detectors.ResultVerifier, + detectorType detector_typepb.DetectorType, + results []detectors.Result, +) ([]detectors.Result, error) { + // Only remote verification is timed: cache hits never reach verifyResult, so a fully cached chunk records nothing. + // The aggregate is summed and reported once per chunk because MetricsReporter implementations such as + // InMemoryMetrics truncate each report to whole milliseconds, which would drop time if reported per call. + var timeSpentVerifying time.Duration + defer func() { + if timeSpentVerifying > 0 { + v.metrics.AddFromDataVerifyTimeSpent(timeSpentVerifying) + } + }() + verifyResult := func(i int) { + verifyStart := time.Now() + detector.VerifyResult(ctx, &results[i]) + elapsed := time.Since(verifyStart) + v.recordDetectorVerifyTime(detectorType, elapsed) + timeSpentVerifying += elapsed + } + + for i := range results { + cacheKey, err := v.getResultCacheKey(results[i]) + if err != nil { + ctx.Logger().Error(err, "error getting result cache key for verification caching", + "operation", "read") + // Fail open: a result we cannot key still deserves verification, matching the + // all-or-nothing path, where a key error falls through to FromData(verify=true). + verifyResult(i) + continue + } + if cacheHit, ok := v.resultCache.Get(string(cacheKey)); ok { + results[i].CopyVerificationInfo(&cacheHit) + results[i].VerificationFromCache = true + v.metrics.AddResultCacheHits(1) + v.metrics.AddCredentialVerificationsSaved(1) + continue + } + v.metrics.AddResultCacheMisses(1) + verifyResult(i) + copyForCaching := results[i] + // Do not persist raw secret values in a long-lived cache + copyForCaching.ClearSecrets() + v.resultCache.Set(string(cacheKey), copyForCaching) + } + + return results, nil +} + func (v *VerificationCache) getResultCacheKey(result detectors.Result) ([]byte, error) { v.hashMu.Lock() defer v.hashMu.Unlock() diff --git a/pkg/verificationcache/verification_cache_test.go b/pkg/verificationcache/verification_cache_test.go index ec6aaecffd9a..3946a2e04cbd 100644 --- a/pkg/verificationcache/verification_cache_test.go +++ b/pkg/verificationcache/verification_cache_test.go @@ -3,6 +3,7 @@ package verificationcache import ( "context" "errors" + "sync" "testing" "time" @@ -17,13 +18,25 @@ import ( type testDetector struct { fromDataCallCount int results []detectors.Result + // fromDataErr, when set, is returned by FromData instead of results, so tests can exercise what + // happens when remote verification fails rather than merely returning an unverified credential. + fromDataErr error } func (t *testDetector) FromData(_ context.Context, verify bool, _ []byte) ([]detectors.Result, error) { t.fromDataCallCount = t.fromDataCallCount + 1 var results []detectors.Result for _, r := range t.results { - copy := detectors.Result{Redacted: r.Redacted, Raw: r.Raw, RawV2: r.RawV2, DetectorType: r.DetectorType} + copy := detectors.Result{ + Redacted: r.Redacted, + Raw: r.Raw, + RawV2: r.RawV2, + DetectorType: r.DetectorType, + SecretParts: r.SecretParts, + } + if v := r.GetPrimarySecretValue(); v != "" { + copy.SetPrimarySecretValue(v) + } if verify { copy.CopyVerificationInfo(&r) } @@ -34,6 +47,12 @@ func (t *testDetector) FromData(_ context.Context, verify bool, _ []byte) ([]det // monitored. time.Sleep(2 * time.Millisecond) + // Failing after the sleep keeps the elapsed time observable, which is what the error-path timing + // test needs to assert. + if t.fromDataErr != nil { + return nil, t.fromDataErr + } + return results, nil } @@ -43,6 +62,129 @@ func (t *testDetector) Description() string { return "" } var _ detectors.Detector = (*testDetector)(nil) +// testResultVerifier is a testDetector that can also verify results one at a time, so +// tests can exercise the verification cache's per-result path. It inherits +// fromDataCallCount, which is what proves that path never re-runs the detector to verify. +type testResultVerifier struct { + testDetector + verifyResultCallCount int + // verifyResultCalls records the Redacted value of each verified result in call order, + // so tests can assert exactly which results were verified rather than just how many. + verifyResultCalls []string +} + +func (t *testResultVerifier) VerifyResult(_ context.Context, result *detectors.Result) { + t.verifyResultCallCount++ + t.verifyResultCalls = append(t.verifyResultCalls, result.Redacted) + + // Stand in for a remote verification by adopting the status the test declared for this + // credential. Results are matched on Redacted because that is how these tests identify + // them; FromData(verify=false) strips verification info, so it has to be restored here. + for i := range t.results { + if t.results[i].Redacted == result.Redacted { + result.CopyVerificationInfo(&t.results[i]) + break + } + } + + // As in testDetector.FromData, the metric timing resolution is 1 ms, so verification has + // to be artificially slow for the wall time it reports to be observable. + time.Sleep(2 * time.Millisecond) +} + +var ( + _ detectors.Detector = (*testResultVerifier)(nil) + _ detectors.ResultVerifier = (*testResultVerifier)(nil) +) + +// typedTestDetector reports a detector type chosen by the test rather than the fixed -1 of +// testDetector, so tests can prove that verification timing is attributed to the detector that +// incurred it and not merely recorded. +type typedTestDetector struct { + testDetector + detectorType detector_typepb.DetectorType +} + +func (t *typedTestDetector) Type() detector_typepb.DetectorType { return t.detectorType } + +var _ detectors.Detector = (*typedTestDetector)(nil) + +// detectorVerifySample is one observation handed to DetectorMetricsReporter. +type detectorVerifySample struct { + detectorType detector_typepb.DetectorType + wallTime time.Duration +} + +// detectorMetricsRecorder is an InMemoryMetrics that additionally opts into per-detector +// verification timing. Keeping the aggregate behavior by embedding is deliberate: these tests have +// to show that the new per-detector view is added without disturbing the existing counter, so both +// have to be observable through one reporter, exactly as thog's reporter will see them. +type detectorMetricsRecorder struct { + InMemoryMetrics + + mu sync.Mutex + // samples accumulates per-detector observations in call order. + samples []detectorVerifySample + // aggregateCalls counts calls to AddFromDataVerifyTimeSpent. The sum alone cannot distinguish + // one report of an accumulated total from several per-call reports, and that distinction is the + // point of the per-remote-call granularity tests. + aggregateCalls int +} + +var ( + _ MetricsReporter = (*detectorMetricsRecorder)(nil) + _ DetectorMetricsReporter = (*detectorMetricsRecorder)(nil) +) + +func (m *detectorMetricsRecorder) AddFromDataVerifyTimeSpent(wallTime time.Duration) { + m.mu.Lock() + m.aggregateCalls++ + m.mu.Unlock() + + m.InMemoryMetrics.AddFromDataVerifyTimeSpent(wallTime) +} + +func (m *detectorMetricsRecorder) AddDetectorVerifyTimeSpent( + detectorType detector_typepb.DetectorType, + wallTime time.Duration, +) { + m.mu.Lock() + defer m.mu.Unlock() + + m.samples = append(m.samples, detectorVerifySample{detectorType: detectorType, wallTime: wallTime}) +} + +func (m *detectorMetricsRecorder) recordedSamples() []detectorVerifySample { + m.mu.Lock() + defer m.mu.Unlock() + + return append([]detectorVerifySample(nil), m.samples...) +} + +func (m *detectorMetricsRecorder) aggregateReportCount() int { + m.mu.Lock() + defer m.mu.Unlock() + + return m.aggregateCalls +} + +// assertSampleTypes asserts that the recorded samples carry exactly the given detector types, in +// order, and that each one reports observable wall time. A sample of zero duration would satisfy a +// count-only assertion while carrying no latency signal at all. +func assertSampleTypes(t *testing.T, recorder *detectorMetricsRecorder, want ...detector_typepb.DetectorType) { + t.Helper() + + samples := recorder.recordedSamples() + // Left nil rather than preallocated so that the no-samples case compares equal to a caller that + // passed no expected types at all. + var gotTypes []detector_typepb.DetectorType + for _, sample := range samples { + gotTypes = append(gotTypes, sample.detectorType) + assert.Greater(t, sample.wallTime, time.Duration(0)) + } + assert.Equal(t, want, gotTypes) +} + func getResultCacheKey(t *testing.T, cache *VerificationCache, result detectors.Result) string { key, err := cache.getResultCacheKey(result) require.NoError(t, err) @@ -280,3 +422,460 @@ func TestVerificationCache_FromData_SameRawV2DifferentType_CacheMiss(t *testing. } assert.Len(t, cache.resultCache.Values(), 2) } + +// The tests below cover the per-result verification path taken by detectors that implement +// detectors.ResultVerifier. +func TestVerificationCache_FromData_ResultVerifier_PartialCacheHit(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }}} + detector.results[1].SetVerificationError(errors.New("test verification error")) + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: true}) + + results, err := cache.FromData( + logContext.Background(), + &detector, + true, + false, + nil) + + require.NoError(t, err) + // One extraction pass and no re-run: the whole point of the per-result path. + assert.Equal(t, 1, detector.fromDataCallCount) + assert.Equal(t, 1, detector.verifyResultCallCount) + assert.Equal(t, []string{"world"}, detector.verifyResultCalls) + wantResults := []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true, + VerificationFromCache: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + } + wantResults[1].SetVerificationError(errors.New("test verification error")) + assert.ElementsMatch(t, wantResults, results) + wantCacheData := []detectors.Result{ + {Redacted: "hello", Verified: true}, + {Redacted: "world", Verified: false}, + } + wantCacheData[1].SetVerificationError(errors.New("test verification error")) + assert.ElementsMatch(t, wantCacheData, cache.resultCache.Values()) + assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) + assert.Equal(t, int32(1), metrics.CredentialVerificationsSaved.Load()) + assert.Equal(t, int32(1), metrics.ResultCacheHits.Load()) + assert.Equal(t, int32(1), metrics.ResultCacheMisses.Load()) + // Nothing is ever discarded on this path, so no hit can be wasted. + assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load()) +} + +func TestVerificationCache_FromData_ResultVerifier_AllCacheHits(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }}} + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cacheData := []detectors.Result{ + {Redacted: "hello", Verified: true}, + {Redacted: "world", Verified: false}, + } + cacheData[1].SetVerificationError(errors.New("test verification error")) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), cacheData[0]) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]), cacheData[1]) + + results, err := cache.FromData( + logContext.Background(), + &detector, + true, + false, + nil) + + require.NoError(t, err) + assert.Equal(t, 1, detector.fromDataCallCount) + assert.Equal(t, 0, detector.verifyResultCallCount) + wantResults := []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true, + VerificationFromCache: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false, + VerificationFromCache: true}, + } + wantResults[1].SetVerificationError(errors.New("test verification error")) + assert.ElementsMatch(t, wantResults, results) + assert.ElementsMatch(t, cacheData, cache.resultCache.Values()) + // A fully cached chunk makes no remote calls, so no verify time may be recorded, + // matching the all-or-nothing path's early return on full cache coverage. + assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) + assert.Equal(t, int32(2), metrics.CredentialVerificationsSaved.Load()) + assert.Equal(t, int32(2), metrics.ResultCacheHits.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load()) +} + +func TestVerificationCache_FromData_ResultVerifier_NoCacheHits(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }}} + detector.results[1].SetVerificationError(errors.New("test verification error")) + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + + results, err := cache.FromData( + logContext.Background(), + &detector, + true, + false, + nil) + + require.NoError(t, err) + assert.Equal(t, 1, detector.fromDataCallCount) + assert.Equal(t, 2, detector.verifyResultCallCount) + assert.Equal(t, []string{"hello", "world"}, detector.verifyResultCalls) + assert.ElementsMatch(t, detector.results, results) + // Raw and RawV2 must be absent from every cached entry: this cache outlives the scan of + // any single chunk, so raw credential material must not be retained in it. + cachedValues := cache.resultCache.Values() + assert.Len(t, cachedValues, 2) + for _, cached := range cachedValues { + assert.Nil(t, cached.Raw) + assert.Nil(t, cached.RawV2) + } + assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) + assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHits.Load()) + assert.Equal(t, int32(2), metrics.ResultCacheMisses.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load()) +} + +// Identical pairs recur within a chunk, so the cache read must happen immediately before +// each verification rather than once up front. Getting this wrong would still be correct, +// just wasteful, which is exactly the class of bug this change exists to remove. +func TestVerificationCache_FromData_ResultVerifier_DuplicateResultsInChunk(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }}} + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + + results, err := cache.FromData( + logContext.Background(), + &detector, + true, + false, + nil) + + require.NoError(t, err) + assert.Equal(t, 1, detector.verifyResultCallCount) + // The first occurrence is verified remotely and populates the cache; the second reads it + // back, which is why it reports VerificationFromCache despite the same scan verifying it. + assert.ElementsMatch(t, []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true, + VerificationFromCache: true}, + }, results) + assert.Len(t, cache.resultCache.Values(), 1) + assert.Equal(t, int32(1), metrics.CredentialVerificationsSaved.Load()) + assert.Equal(t, int32(1), metrics.ResultCacheHits.Load()) + assert.Equal(t, int32(1), metrics.ResultCacheMisses.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load()) +} + +// Implementing ResultVerifier must not make a detector verify when verification is off, so +// the dispatch has to sit after FromData's verify=false early return. +func TestVerificationCache_FromData_ResultVerifier_VerifyFalse(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }}} + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + + results, err := cache.FromData( + logContext.Background(), + &detector, + false, + false, + nil) + + require.NoError(t, err) + assert.Equal(t, 1, detector.fromDataCallCount) + assert.Equal(t, 0, detector.verifyResultCallCount) + assert.ElementsMatch(t, []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false}, + }, results) + assert.Empty(t, cache.resultCache.Values()) + assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHits.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load()) +} + +// Targeted re-verification scans (chunk.SecretID != 0) deliberately bypass the cache to get +// a fresh answer, so they must keep going through FromData(verify=true) even for a detector +// that could verify per result. +func TestVerificationCache_FromData_ResultVerifier_ForceCacheUpdate(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }}} + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: false}) + + results, err := cache.FromData( + logContext.Background(), + &detector, + true, + true, + nil) + + require.NoError(t, err) + assert.Equal(t, 1, detector.fromDataCallCount) + assert.Equal(t, 0, detector.verifyResultCallCount) + assert.ElementsMatch(t, detector.results, results) + // The stale cached entry is replaced by the freshly verified status. + assert.ElementsMatch(t, []detectors.Result{{Redacted: "hello", Verified: true}}, + cache.resultCache.Values()) + assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHits.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load()) + assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load()) +} + +func TestVerificationCache_FromData_DoesNotCacheSecretMaterial(t *testing.T) { + result := detectors.Result{ + Redacted: "hello", + Raw: []byte("hello"), + RawV2: []byte("helloV2"), + Verified: true, + SecretParts: map[string]string{"key": "hello"}, + } + result.SetPrimarySecretValue("hello") + detector := testDetector{results: []detectors.Result{result}} + cache := New(simple.NewCache[detectors.Result](), nil) + + results, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + require.NoError(t, err) + + require.Len(t, results, 1) + assert.Equal(t, []byte("hello"), results[0].Raw) + assert.Equal(t, []byte("helloV2"), results[0].RawV2) + assert.Equal(t, map[string]string{"key": "hello"}, results[0].SecretParts) + assert.Equal(t, "hello", results[0].GetPrimarySecretValue()) + + cached := cache.resultCache.Values() + require.Len(t, cached, 1) + assert.Nil(t, cached[0].Raw) + assert.Nil(t, cached[0].RawV2) + assert.Nil(t, cached[0].SecretParts) + assert.Empty(t, cached[0].GetPrimarySecretValue()) +} + +// The tests below cover per-detector verification timing, which a reporter receives only by also +// implementing DetectorMetricsReporter. They exist to catch two classes of regression: a sample +// attributed to the wrong detector, and a sample recorded for verification that never left the +// process because the result cache answered it. + +func TestVerificationCache_FromData_PerDetectorTiming_Passthrough(t *testing.T) { + detector := testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }} + metrics := detectorMetricsRecorder{} + cache := New(nil, &metrics) + + _, err := cache.FromData(logContext.Background(), &detector, true, true, nil) + + require.NoError(t, err) + assertSampleTypes(t, &metrics, detector.Type()) + assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) +} + +func TestVerificationCache_FromData_PerDetectorTiming_PassthroughVerifyFalse(t *testing.T) { + detector := testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }} + metrics := detectorMetricsRecorder{} + cache := New(nil, &metrics) + + _, err := cache.FromData(logContext.Background(), &detector, false, true, nil) + + require.NoError(t, err) + assertSampleTypes(t, &metrics) + assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) +} + +func TestVerificationCache_FromData_PerDetectorTiming_CacheMiss(t *testing.T) { + detector := testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }} + metrics := detectorMetricsRecorder{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: true}) + + _, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + + require.NoError(t, err) + // The uncached finding forces one FromData(verify=true) pass over the whole chunk, which is one + // verification pass and therefore one sample. + assertSampleTypes(t, &metrics, detector.Type()) +} + +// A chunk answered entirely from the cache issues no remote request, so recording a duration for it +// would put fabricated latency into the histogram. +func TestVerificationCache_FromData_PerDetectorTiming_AllCacheHits(t *testing.T) { + detector := testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }} + metrics := detectorMetricsRecorder{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: true}) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]), + detectors.Result{Redacted: "world", Verified: false}) + + _, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + + require.NoError(t, err) + assertSampleTypes(t, &metrics) + assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) +} + +func TestVerificationCache_FromData_PerDetectorTiming_ForceCacheUpdate(t *testing.T) { + detector := testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }} + metrics := detectorMetricsRecorder{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: false}) + + _, err := cache.FromData(logContext.Background(), &detector, true, true, nil) + + require.NoError(t, err) + // Targeted rescans bypass the cache to get a fresh answer, so they do make a remote call. + assertSampleTypes(t, &metrics, detector.Type()) +} + +func TestVerificationCache_FromData_PerDetectorTiming_ResultVerifierPartialCacheHit(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }}} + metrics := detectorMetricsRecorder{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: true}) + + _, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + + require.NoError(t, err) + // Only the miss is verified remotely, so the cache hit contributes no sample. + require.Equal(t, 1, detector.verifyResultCallCount) + assertSampleTypes(t, &metrics, detector.Type()) +} + +func TestVerificationCache_FromData_PerDetectorTiming_ResultVerifierAllCacheHits(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }}} + metrics := detectorMetricsRecorder{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), + detectors.Result{Redacted: "hello", Verified: true}) + cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]), + detectors.Result{Redacted: "world", Verified: false}) + + _, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + + require.NoError(t, err) + require.Equal(t, 0, detector.verifyResultCallCount) + assertSampleTypes(t, &metrics) + assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) +} + +// One sample per remote call is what makes the histogram readable as provider latency: a chunk with +// several misses must not collapse into a single fat observation. The aggregate counter keeps its +// existing shape of one accumulated report per chunk, because it truncates to whole milliseconds and +// would lose time if it were reported per call. +func TestVerificationCache_FromData_PerDetectorTiming_ResultVerifierSamplesPerRemoteCall(t *testing.T) { + detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false}, + }}} + metrics := detectorMetricsRecorder{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + + _, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + + require.NoError(t, err) + require.Equal(t, 2, detector.verifyResultCallCount) + assertSampleTypes(t, &metrics, detector.Type(), detector.Type()) + assert.Equal(t, 1, metrics.aggregateReportCount()) +} + +func TestVerificationCache_FromData_PerDetectorTiming_AttributesToOwnDetectorType(t *testing.T) { + const ( + firstType detector_typepb.DetectorType = -10 + secondType detector_typepb.DetectorType = -11 + ) + first := typedTestDetector{ + testDetector: testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), Verified: true}, + }}, + detectorType: firstType, + } + second := typedTestDetector{ + testDetector: testDetector{results: []detectors.Result{ + {Redacted: "world", Raw: []byte("world"), Verified: true}, + }}, + detectorType: secondType, + } + metrics := detectorMetricsRecorder{} + cache := New(nil, &metrics) + + _, err := cache.FromData(logContext.Background(), &first, true, true, nil) + require.NoError(t, err) + _, err = cache.FromData(logContext.Background(), &second, true, true, nil) + require.NoError(t, err) + + assertSampleTypes(t, &metrics, firstType, secondType) +} + +// A reporter that predates this interface, or simply does not want the extra series, must keep +// working: New leaves detectorMetrics nil and the timing sites have to tolerate that. +func TestVerificationCache_FromData_PerDetectorTiming_ReporterWithoutOptIn(t *testing.T) { + detector := testDetector{results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }} + metrics := InMemoryMetrics{} + cache := New(simple.NewCache[detectors.Result](), &metrics) + require.Nil(t, cache.detectorMetrics) + + _, err := cache.FromData(logContext.Background(), &detector, true, false, nil) + + require.NoError(t, err) + assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load()) +} + +// A provider that has started timing out or erroring is precisely the shift this metric exists to +// surface, so a failed verification still has to contribute its latency. +func TestVerificationCache_FromData_PerDetectorTiming_RecordsOnVerificationError(t *testing.T) { + detector := testDetector{ + results: []detectors.Result{ + {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true}, + }, + fromDataErr: errors.New("test verification failure"), + } + metrics := detectorMetricsRecorder{} + cache := New(nil, &metrics) + + _, err := cache.FromData(logContext.Background(), &detector, true, true, nil) + + require.Error(t, err) + assertSampleTypes(t, &metrics, detector.Type()) +} diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter.go b/pkg/writers/buffered_file_writer/bufferedfilewriter.go index 49c3afa9a8e8..e603f1580984 100644 --- a/pkg/writers/buffered_file_writer/bufferedfilewriter.go +++ b/pkg/writers/buffered_file_writer/bufferedfilewriter.go @@ -132,7 +132,7 @@ func (w *BufferedFileWriter) String() (string, error) { if err != nil { return "", fmt.Errorf("failed to open file: %w", err) } - defer file.Close() + defer func() { _ = file.Close() }() var buf bytes.Buffer // Read the file contents into the buffer. @@ -312,6 +312,6 @@ func newAutoDeletingFileReader(file *os.File) *autoDeletingFileReader { // Close implements the io.Closer interface, deletes the file after closing. func (r *autoDeletingFileReader) Close() error { - defer os.Remove(r.Name()) // Delete the file after closing + defer func() { _ = os.Remove(r.Name()) }() // Delete the file after closing return r.File.Close() } diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go b/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go index 193c398f27cb..61863dbd8cb9 100644 --- a/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go +++ b/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go @@ -118,7 +118,7 @@ func BenchmarkBufferedFileWriterString_BufferOnly_Small(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } func BenchmarkBufferedFileWriterString_BufferOnly_Medium(b *testing.B) { @@ -135,7 +135,7 @@ func BenchmarkBufferedFileWriterString_BufferOnly_Medium(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } func BenchmarkBufferedFileWriterString_OnlyFile_Small(b *testing.B) { @@ -153,7 +153,7 @@ func BenchmarkBufferedFileWriterString_OnlyFile_Small(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } func BenchmarkBufferedFileWriterString_OnlyFile_Medium(b *testing.B) { @@ -171,7 +171,7 @@ func BenchmarkBufferedFileWriterString_OnlyFile_Medium(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } func BenchmarkBufferedFileWriterString_BufferWithFile_Small(b *testing.B) { @@ -193,7 +193,7 @@ func BenchmarkBufferedFileWriterString_BufferWithFile_Small(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } func BenchmarkBufferedFileWriterString_BufferWithFile_Medium(b *testing.B) { @@ -215,7 +215,7 @@ func BenchmarkBufferedFileWriterString_BufferWithFile_Medium(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } func benchmarkBufferedFileWriterString(b *testing.B, w *BufferedFileWriter) { @@ -458,7 +458,7 @@ func BenchmarkBufferedFileWriterWriteLarge(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } } @@ -487,7 +487,7 @@ func BenchmarkBufferedFileWriterWriteSmall(b *testing.B) { rc, err := writer.ReadCloser() assert.NoError(b, err) - rc.Close() + _ = rc.Close() } } @@ -512,7 +512,7 @@ func TestBufferWriterCloseForWritingWithFile(t *testing.T) { rdr, err := writer.ReadCloser() assert.NoError(t, err) - defer rdr.Close() + defer func() { _ = rdr.Close() }() // Get a buffer from the pool and check if it is the same buffer used in the writer. bufFromPool := bufPool.Get() @@ -584,7 +584,7 @@ func TestBufferedFileWriter_ReadFrom(t *testing.T) { rc, err := writer.ReadCloser() assert.NoError(t, err) - defer rc.Close() + defer func() { _ = rc.Close() }() var result bytes.Buffer diff --git a/proto/credentials.proto b/proto/credentials.proto index efcfa439b300..d105234b3a82 100644 --- a/proto/credentials.proto +++ b/proto/credentials.proto @@ -84,3 +84,9 @@ message GoogleDriveDWD { repeated string include_users = 3; repeated string exclude_users = 4; } + +message BitbucketApiToken { + string email = 1; + string token = 2; + string username = 3; // optional; falls back to Bitbucket's documented alias for git ops when omitted +} \ No newline at end of file diff --git a/proto/custom_detectors.proto b/proto/custom_detectors.proto index 6c72a415ba8f..ebc0ea548ae3 100644 --- a/proto/custom_detectors.proto +++ b/proto/custom_detectors.proto @@ -30,6 +30,7 @@ message VerifierConfig { bool unsafe = 2; repeated string headers = 3; repeated string successRanges = 4; + repeated string rotatedRanges = 5; } message ValidationConfig { diff --git a/proto/detector_type.proto b/proto/detector_type.proto index ddd4c83c654f..b8ed9ffeb703 100644 --- a/proto/detector_type.proto +++ b/proto/detector_type.proto @@ -950,7 +950,7 @@ enum DetectorType { VagrantCloudPersonalToken = 946; BetterStack = 947; ZeroTier = 948; - AppOptics = 949; + AppOptics = 949 [deprecated = true]; Metabase = 950; CoinbaseWaaS = 951 [deprecated = true]; LemonSqueezy = 952; @@ -1023,7 +1023,7 @@ enum DetectorType { AzureAPIManagementSubscriptionKey = 1019; Harness = 1020; Langfuse = 1021; - BingSubscriptionKey = 1022; + BingSubscriptionKey = 1022 [deprecated = true]; XAI = 1023; AzureDirectManagementKey = 1024; AzureAppConfigConnectionString = 1025; @@ -1049,4 +1049,28 @@ enum DetectorType { BitbucketDataCenter = 1045; JiraDataCenterPAT = 1046; ConfluenceDataCenter = 1047; + Cloudinary = 1048; + Pinecone = 1049; + GitLabOauth2 = 1050; + SpectralOps = 1051; + AWSAppSync = 1052; + BrainTrustApiKey = 1053; + PgAnalyzeReadKey = 1054; + RedHatPyxis = 1055; + OctopusDeploy = 1056; + OpenRouter = 1057; + NewRelicInsightsInsertKey = 1058; + DuffelToken = 1059; + Shippo = 1060; + HashiCorpVaultBatchToken = 1061; + HashiCorpVaultToken = 1062; + Duo = 1063; + NewRelicLicenseKey = 1064; + NewRelicBrowserKey = 1065; + NewRelicUserKey = 1066; + NewRelicInsightsQueryKey = 1067; + NewRelicMobileAppToken = 1068; + SolarWindsObservability = 1069; + HumioAPIToken = 1070; + Resend = 1071; } diff --git a/proto/source_metadata.proto b/proto/source_metadata.proto index 40846c1f7e04..f4ffff0a1482 100644 --- a/proto/source_metadata.proto +++ b/proto/source_metadata.proto @@ -5,6 +5,11 @@ package source_metadata; option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"; import "google/protobuf/timestamp.proto"; +import "validate/validate.proto"; + +message AWSSecretsManager { + string secret_arn = 1 [(validate.rules).string.pattern = "^arn:aws[^:]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+$"]; +} message Azure { string container = 1; @@ -274,6 +279,8 @@ message SharePoint { int64 views = 5; string docid = 6; string email = 7; + int64 version = 8; + int64 comment_number = 9; } message GoogleDrive { @@ -430,5 +437,6 @@ message MetaData { Stdin stdin = 34; SlackContinuous slackContinuous = 35; JSONEnumerator jsonEnumerator = 36; + AWSSecretsManager awsSecretsManager = 37; } } diff --git a/proto/sources.proto b/proto/sources.proto index cf64caca8687..114bf4550764 100644 --- a/proto/sources.proto +++ b/proto/sources.proto @@ -55,6 +55,7 @@ enum SourceType { SOURCE_TYPE_STDIN = 40; SOURCE_TYPE_SLACK_CONTINUOUS = 41; SOURCE_TYPE_JSON_ENUMERATOR = 42; + SOURCE_TYPE_AWS_SECRETS_MANAGER = 43; } message LocalSource { @@ -107,7 +108,7 @@ message Bitbucket { string token = 2; credentials.Oauth2 oauth = 3; credentials.BasicAuth basic_auth = 4; - + credentials.BitbucketApiToken api_token = 14; } repeated string repositories = 5; repeated string ignore_repos = 6; @@ -281,6 +282,9 @@ message GitHub { bool no_cleanup = 23; bool ignore_gists = 24; bool print_legacy_json = 25; + // When using GitHub App credentials, enumerate repositories from every app installation. + bool scan_all_installations = 26; + bool exclude_archived = 27; } message GitHubExperimental { @@ -336,6 +340,10 @@ message Huggingface { bool skip_all_datasets = 18; bool include_discussions = 11; bool include_prs = 19; + repeated string buckets = 20; + repeated string ignore_buckets = 21; + repeated string include_buckets = 22; + bool skip_all_buckets = 23; } enum JiraInstallationType { @@ -385,6 +393,25 @@ message S3 { repeated string roles = 7; repeated string ignore_buckets = 8; bool enable_resumption = 9 [deprecated = true]; + // endpoint targets an S3-compatible service other than AWS S3. When set, + // per-bucket region discovery is skipped because it is an AWS-only API. + string endpoint = 10 [(validate.rules).string.uri_ref = true]; + // region is the region used to sign requests. Defaults to us-east-1. + string region = 11; + // include_prefixes limits the scan to objects whose key matches one of these + // key prefixes. Leave empty to consider every object in the bucket. + repeated string include_prefixes = 12; + // exclude_prefixes skips objects whose key matches one of these key prefixes. + repeated string exclude_prefixes = 13; + // include_extensions limits the scan to objects whose key ends in one of these + // file extensions, written without a leading dot (e.g. "tf", "yaml"). Leave + // empty to consider every extension. Cannot be combined with + // exclude_extensions. + repeated string include_extensions = 14; + // exclude_extensions skips objects whose key ends in one of these file + // extensions, written without a leading dot (e.g. "zip", "mp4"). Cannot be + // combined with include_extensions. + repeated string exclude_extensions = 15; } message Slack { @@ -467,6 +494,9 @@ message Sharepoint { credentials.Oauth2 oauth = 1; } string site_url = 2; + bool include_versions = 3; + bool include_comments = 4; + bool include_attachments = 5; } message AzureRepos { diff --git a/scripts/lint.sh b/scripts/lint.sh index 9e889b2e3517..de5030ee22fb 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -2,10 +2,9 @@ set -euo pipefail # NOTE: Version and args must match .github/workflows/lint.yml -GOLANGCI_LINT_VERSION="v2.11.4" +GOLANGCI_LINT_VERSION="v2.13.2" -# TODO: Re-enable errcheck and staticcheck once pre-existing issues are resolved. -LINT_ARGS="--disable errcheck,staticcheck --enable bodyclose,copyloopvar,misspell --timeout 10m" +LINT_ARGS="--enable bodyclose,copyloopvar,misspell --timeout 10m" GOBIN="$(go env GOPATH)/bin" GOLANGCI_LINT="${GOBIN}/golangci-lint" diff --git a/scripts/test/detect_changed_detectors.sh b/scripts/test/detect_changed_detectors.sh new file mode 100755 index 000000000000..1e1bfbc7f2d1 --- /dev/null +++ b/scripts/test/detect_changed_detectors.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# detect_changed_detectors.sh — Phase 2 +# +# Emits the list of detectors changed between two git refs, formatted for +# trufflehog's --include-detectors flag (comma-separated, lowercase protobuf +# enum names, optional ".v" version suffix). +# +# Source of truth for each detector's identifier: +# - Proto enum name comes from the detector's Type() implementation in its +# source files (e.g. `return detectorspb.DetectorType_AzureBatch` → +# `azurebatch`). Necessary because the package directory often differs +# from the enum name (azure_batch vs AzureBatch, npmtokenv2 vs NpmToken, +# close vs closecrm, etc.). +# - Version comes from the directory suffix only (`/v`). Detectors that +# encode the version in the dir name (e.g. `npmtokenv2`) are emitted +# without a version suffix; trufflehog then matches all versions of that +# proto type — wider scope but correct. +# +# "New detector" detection compares pkg/engine/defaults/defaults.go imports +# between the two refs. A detector imported at HEAD but not at BASE is new. +# +# Modes: +# (none) List all changed detectors at HEAD, one per line, in +# [.v] form. +# --pr-csv Same set as default mode, comma-joined. +# --main-csv Changed detectors that also exist at BASE (excludes new), +# comma-joined. Use as --include-detectors for the main build. +# --new-only Just the new detectors (in HEAD but not BASE), one per line. +# +# Env: +# BASE_REF default origin/main +# HEAD_REF default HEAD + +set -euo pipefail + +MODE="${1:-list}" +BASE_REF="${BASE_REF:-origin/main}" +HEAD_REF="${HEAD_REF:-HEAD}" + +REPO_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPO_ROOT" + +# Resolve BASE to a concrete commit. Workflow already runs `git fetch origin +# main`; locally that may not be true, so we fall back to `main` if the +# remote-tracking ref is missing. +if ! git rev-parse --verify "$BASE_REF" >/dev/null 2>&1; then + if git rev-parse --verify main >/dev/null 2>&1; then + BASE_REF=main + else + echo "error: cannot resolve BASE_REF=$BASE_REF and no local 'main'" >&2 + exit 1 + fi +fi + +MERGE_BASE=$(git merge-base "$BASE_REF" "$HEAD_REF") + +# Step 1 — changed detector dirs (relative to repo root). +# Pattern: pkg/detectors/(/v)?/.go, excludes _test.go and +# files inside common/, custom_detectors/. +mapfile -t CHANGED_DIRS < <( + git diff --name-only "$MERGE_BASE...$HEAD_REF" -- 'pkg/detectors/**/*.go' \ + | grep -Ev '_test\.go$' \ + | grep -Ev '^pkg/detectors/(common|custom_detectors)/' \ + | sed -E 's|^(pkg/detectors/[^/]+(/v[0-9]+)?)/[^/]+\.go$|\1|' \ + | sort -u +) + +# Step 2 — defaults.go imports at each ref. Each line has form +# "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/(/v)?" +# We extract just the (/v)? portion to use as the dir identifier. +parse_defaults_imports() { + local ref="$1" + git show "$ref:pkg/engine/defaults/defaults.go" 2>/dev/null \ + | grep -oE '"github\.com/trufflesecurity/trufflehog/v3/pkg/detectors/[^"]+"' \ + | sed -E 's|.*/pkg/detectors/||; s|"$||' \ + | sort -u +} + +mapfile -t HEAD_IMPORTS < <(parse_defaults_imports "$HEAD_REF") +mapfile -t BASE_IMPORTS < <(parse_defaults_imports "$MERGE_BASE") + +# Set difference: detectors imported at HEAD but not at BASE. The dir +# identifier (e.g. "github/v2", "stripe") matches the form we extracted in +# step 1, so we can intersect directly without re-mapping. +NEW_DIRS_FILE=$(mktemp) +trap 'rm -f "$NEW_DIRS_FILE"' EXIT +comm -23 \ + <(printf '%s\n' "${HEAD_IMPORTS[@]+"${HEAD_IMPORTS[@]}"}") \ + <(printf '%s\n' "${BASE_IMPORTS[@]+"${BASE_IMPORTS[@]}"}") \ + > "$NEW_DIRS_FILE" + +is_new_detector() { + grep -qxF "$1" "$NEW_DIRS_FILE" +} + +# Step 2b — skip detectors whose diff doesn't touch regex patterns or Keywords. +# Corpora results only change when the matching logic changes; verification, +# redaction, or structural changes don't affect match counts. +has_pattern_change() { + local dir="$1" + + # Fast path: regex or Keywords() signature on a changed line. + git diff "$MERGE_BASE...$HEAD_REF" -- "$dir"/*.go 2>/dev/null \ + | grep -qE '^[+-][^+-].*(regexp\.|MustCompile|Keywords)' && return 0 + + # Slow path: compare the Keywords() function body between refs to catch + # changes to the return value (e.g. []string{"old"} → []string{"new"}) + # where the changed lines don't mention "Keywords" themselves. + local file + while IFS= read -r file; do + [[ "$file" == *_test.go ]] && continue + local head_body base_body + head_body=$(git show "$HEAD_REF:$file" 2>/dev/null \ + | awk '/func[[:space:]].*Keywords\(\)[[:space:]]*\[\]string/,/^[[:space:]]*\}/' \ + | tail -n +2) + base_body=$(git show "$MERGE_BASE:$file" 2>/dev/null \ + | awk '/func[[:space:]].*Keywords\(\)[[:space:]]*\[\]string/,/^[[:space:]]*\}/' \ + | tail -n +2) + [[ "$head_body" != "$base_body" ]] && return 0 + done < <(git diff --name-only "$MERGE_BASE...$HEAD_REF" -- "$dir"/*.go 2>/dev/null) + + return 1 +} + +# Step 3 — for a dir, derive `[.v]`. +detector_id_for_dir() { + local dir="$1" + local version="" + if [[ "$dir" =~ ^pkg/detectors/[^/]+/v([0-9]+)$ ]]; then + version=".v${BASH_REMATCH[1]}" + fi + + # Extract proto enum name. Multiple matches are possible (a detector may + # also reference related types in helpers); the Type() return is by far + # the most common, so the modal value wins. + local proto + proto=$( + grep -E 'return[[:space:]]+\S*DetectorType_[A-Za-z0-9]+' "$dir"/*.go 2>/dev/null \ + | grep -v '_test\.go' \ + | grep -oE 'DetectorType_[A-Za-z0-9]+' \ + | sort | uniq -c | sort -rn \ + | head -1 \ + | awk '{print $2}' \ + | sed 's/^DetectorType_//' \ + | tr '[:upper:]' '[:lower:]' + ) + if [[ -z "$proto" ]]; then + return 1 + fi + echo "${proto}${version}" +} + +# Step 4 — emit per mode. +emit_list() { + local dir id + for dir in "${CHANGED_DIRS[@]:-}"; do + [[ -z "$dir" ]] && continue + has_pattern_change "$dir" || continue + if id=$(detector_id_for_dir "$dir"); then + echo "$id" + else + echo "warning: could not resolve detector id for $dir" >&2 + fi + done | sort -u +} + +emit_main_list() { + local dir id + for dir in "${CHANGED_DIRS[@]:-}"; do + [[ -z "$dir" ]] && continue + has_pattern_change "$dir" || continue + # Strip `pkg/detectors/` prefix to get the import-path form, then + # check against the new-detector set. + local import_form="${dir#pkg/detectors/}" + if is_new_detector "$import_form"; then + continue + fi + if id=$(detector_id_for_dir "$dir"); then + echo "$id" + fi + done | sort -u +} + +emit_new_list() { + local dir id + for dir in "${CHANGED_DIRS[@]:-}"; do + [[ -z "$dir" ]] && continue + has_pattern_change "$dir" || continue + local import_form="${dir#pkg/detectors/}" + if ! is_new_detector "$import_form"; then + continue + fi + if id=$(detector_id_for_dir "$dir"); then + echo "$id" + fi + done | sort -u +} + +case "$MODE" in + list) emit_list ;; + --pr-csv) emit_list | paste -sd, - ;; + --main-csv) emit_main_list | paste -sd, - ;; + --new-only) emit_new_list ;; + *) echo "Usage: $0 [--pr-csv|--main-csv|--new-only]" >&2; exit 2 ;; +esac diff --git a/scripts/test/detector_corpora_test.sh b/scripts/test/detector_corpora_test.sh new file mode 100755 index 000000000000..9b17eacfcb79 --- /dev/null +++ b/scripts/test/detector_corpora_test.sh @@ -0,0 +1,127 @@ +#!/bin/bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [ ...]" + exit 1 +fi + +# CI sets OUTPUT_JSONL to per-run paths and skips the human-readable DuckDB +# summary. Local invocations leave it unset and get the summary table for +# debugging. +if [[ -z "${OUTPUT_JSONL+x}" ]]; then + OUTPUT_JSONL="/tmp/corpora_results.jsonl" + RUN_DUCKDB_SUMMARY=1 +else + RUN_DUCKDB_SUMMARY=0 +fi +> "$OUTPUT_JSONL" + +REPO_ROOT="$(git rev-parse --show-toplevel)" +TRUFFLEHOG_BIN="${TRUFFLEHOG_BIN:-${REPO_ROOT}/trufflehog}" + +if [[ ! -x "$TRUFFLEHOG_BIN" ]]; then + CGO_ENABLED=0 go build -o "$TRUFFLEHOG_BIN" "$REPO_ROOT" +fi + +# When set, scope the scan to specific detectors. Comma-separated, lowercase +# proto enum names with optional ".v" suffix (matches the format produced +# by scripts/test/detect_changed_detectors.sh). +INCLUDE_DETECTORS="${INCLUDE_DETECTORS:-}" +INCLUDE_FLAG=() +if [[ -n "$INCLUDE_DETECTORS" ]]; then + INCLUDE_FLAG=(--include-detectors="$INCLUDE_DETECTORS") +fi + +if [[ -n "${OUTPUT_JSONL_MAIN:-}" ]]; then + > "$OUTPUT_JSONL_MAIN" +fi + +# --no-verification avoids network calls against a large corpus where thousands +# of matches could trigger API calls, dominating runtime. Verifier behavior is +# covered by detector unit and integration tests. +# +# Dual-binary mode: when TRUFFLEHOG_BIN_MAIN / OUTPUT_JSONL_MAIN / +# INCLUDE_DETECTORS_MAIN are set, the corpus stream is teed to both the PR +# binary (stdout side) and the main binary (process substitution) so S3 is +# only downloaded once. +scan() { + local input="$1" + set +e + + local main_include_flag=() + if [[ -n "${INCLUDE_DETECTORS_MAIN:-}" ]]; then + main_include_flag=(--include-detectors="$INCLUDE_DETECTORS_MAIN") + fi + + local rc=0 + if [[ -n "${TRUFFLEHOG_BIN_MAIN:-}" ]]; then + # Single S3 download teed to both binaries simultaneously. + unzstd -c "$input" \ + | jq -r .content \ + | tee >( + "${TRUFFLEHOG_BIN_MAIN}" \ + --no-update \ + --no-verification \ + --allow-verification-overlap \ + --log-level=3 \ + --concurrency=8 \ + --json \ + --archive-timeout=2h \ + "${main_include_flag[@]}" \ + stdin >> "${OUTPUT_JSONL_MAIN}" + ) \ + | "$TRUFFLEHOG_BIN" \ + --no-update \ + --no-verification \ + --allow-verification-overlap \ + --log-level=3 \ + --concurrency=8 \ + --json \ + --print-avg-detector-time \ + --archive-timeout=2h \ + "${INCLUDE_FLAG[@]}" \ + stdin >> "$OUTPUT_JSONL" + rc=$? + wait + else + unzstd -c "$input" \ + | jq -r .content \ + | "$TRUFFLEHOG_BIN" \ + --no-update \ + --no-verification \ + --allow-verification-overlap \ + --log-level=3 \ + --concurrency=8 \ + --json \ + --print-avg-detector-time \ + --archive-timeout=2h \ + "${INCLUDE_FLAG[@]}" \ + stdin >> "$OUTPUT_JSONL" + rc=$? + fi + set -e + return $rc +} + +for CORPORA_FILE in "$@"; do + if [[ "$CORPORA_FILE" == s3://* ]]; then + aws s3 cp "$CORPORA_FILE" - | scan /dev/stdin + else + scan "$CORPORA_FILE" + fi +done + +if [[ "$RUN_DUCKDB_SUMMARY" == "1" ]]; then + duckdb -c " +CREATE TABLE t AS FROM read_json_auto('$OUTPUT_JSONL', ignore_errors=true); + +SELECT + t.DetectorName detector, + COUNT(*) total +FROM t +GROUP BY all +ORDER BY total DESC, detector +LIMIT 50; +" +fi diff --git a/scripts/test/diff_corpora_results.py b/scripts/test/diff_corpora_results.py new file mode 100755 index 000000000000..1362bf8b3060 --- /dev/null +++ b/scripts/test/diff_corpora_results.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +Diffs two trufflehog JSONL outputs (main vs PR build) and emits a Markdown +report to stdout. + +Identity per finding: (DetectorName, Raw or RawV2 fallback). Set semantics — +duplicates within a single scan collapse into one identity, so a regex change +either adds a new (detector, secret) identity or removes one. + +Verification is disabled at scan time (--no-verification) to avoid network +calls against a large corpus where thousands of matches could dominate runtime. +The diff measures regex match changes only. + +When --changed-detectors is provided, the report focuses on the detectors +changed by the PR. Detectors flagged via --new-detectors are rendered with 🆕 +status and absolute density (no main baseline). A scoped detector that is not +new and has zero matches on both sides renders with ⚪: the corpus holds no +material for it. When --corpus-bytes is provided, a blast-radius column projects matches +per 10 GB of scanned content. + +Usage: + diff_corpora_results.py + [--changed-detectors=] + [--new-detectors=] + [--corpus-bytes=] +""" +import argparse +import json +import sys +from collections import defaultdict + + +PREAMBLE = ( + "Scans a corpus of real-world public code against only the detectors " + "changed in this PR, then compares unique match counts between the PR " + "build and the main baseline to catch regex regressions. Verification " + "is disabled — each detector's regex is measured independently." +) + +STATUS_KEY = ( + "- 🔴 regression: >5 new, >20% increase over main, or any removed\n" + "- ⚠️ warning: 1–5 new and ≤20% increase over main\n" + "- ✅ clean\n" + "- ⚪ no data: 0 matches on main and PR; no comparison possible\n" + "- 🆕 new detector (no baseline)" +) + +# Marker on the very first line of the body so peter-evans/find-comment can +# locate the sticky comment via substring match. Workflow file references the +# same literal — keep the two in sync. +STICKY_COMMENT_MARKER = "" + + +def parse_csv(s): + """Parse a comma-separated detector list into normalized name set. + + Strips ``.v`` version suffixes and lowercases. JSONL DetectorName is the + proto enum name (e.g., ``JDBC``); we match case-insensitively by name only, + since version doesn't appear in the output. Versioned scoping happens at + the trufflehog --include-detectors level. + """ + if not s: + return set() + out = set() + for item in s.split(","): + item = item.strip() + if not item: + continue + if "." in item: + item = item.split(".", 1)[0] + out.add(item.lower()) + return out + + +def load_findings(path): + """Returns dict: detector_name -> {"identities": set[str], "total": int}.""" + by_detector = defaultdict(lambda: {"identities": set(), "total": 0}) + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + detector = obj.get("DetectorName") or "" + if not detector: + continue + raw = obj.get("Raw") or obj.get("RawV2") or "" + by_detector[detector]["identities"].add(raw) + by_detector[detector]["total"] += 1 + return by_detector + + +def status_emoji(new_count, removed_count, unique_main): + """Hybrid threshold: 🔴 on absolute (>5) OR relative (>20% of main) NEW, OR any REMOVED.""" + if removed_count > 0: + return "🔴" + if new_count > 5 or new_count > 0.20 * max(unique_main, 1): + return "🔴" + if new_count > 0: + return "⚠️" + return "✅" + + +def build_top_line_summary(rows, changed): + regressed = sum(1 for r in rows if not r["is_new"] and r["emoji"] == "🔴") + warned = sum(1 for r in rows if not r["is_new"] and r["emoji"] == "⚠️") + new_count = sum(1 for r in rows if r["is_new"]) + clean = sum(1 for r in rows if r["emoji"] == "✅") + no_data = sum(1 for r in rows if r["emoji"] == "⚪") + scoped = ", ".join(f"`{d}`" for d in sorted(changed)) if changed else "" + parts = [] + if regressed: + parts.append(f"{regressed} regressed") + if warned: + parts.append(f"{warned} warned") + parts += [f"{new_count} new", f"{clean} clean"] + if no_data: + parts.append(f"{no_data} no data") + summary = f"**{' · '.join(parts)}**" + if scoped: + summary += f" \u00a0|\u00a0 Scoped to: {scoped}" + return summary + + +def render(main, pr, changed=None, new_detectors=None): + new_detectors = new_detectors or set() + + if changed: + all_names = {d for d in (set(main) | set(pr)) if d.lower() in changed} + # Detectors that the PR claims to have changed (or added) but that + # produced zero matches on either side don't appear in JSONL, so we + # add them explicitly so they get a 0/0 row in the table. + all_names |= changed - {d.lower() for d in all_names} + else: + all_names = set(main) | set(pr) + + _empty = {"identities": set(), "total": 0} + rows = [] + has_diff = False + for d in sorted(all_names): + # A detector is only treated as fully new if the new_detectors set + # says so AND main produced no findings for it. When a PR modifies an + # existing version and adds a new version of the same detector (e.g. + # jdbc.v1 + jdbc.v2), both collapse to "jdbc" in new_detectors but + # main still ran against the existing version — its results must not + # be discarded. + is_new = d.lower() in new_detectors and d not in main + m = main.get(d, _empty) + p = pr.get(d, _empty) + new_ids = p["identities"] - m["identities"] + removed_ids = m["identities"] - p["identities"] + + if is_new: + emoji = "🆕" + elif not m["identities"] and not p["identities"]: + # Zero matches on both sides means the corpus had nothing to compare + emoji = "⚪" + else: + emoji = status_emoji(len(new_ids), len(removed_ids), len(m["identities"])) + + if new_ids or removed_ids or m["total"] != p["total"]: + has_diff = True + + rows.append({ + "detector": d, + "is_new": is_new, + "emoji": emoji, + "total_main": m["total"], + "total_pr": p["total"], + "unique_main": len(m["identities"]), + "unique_pr": len(p["identities"]), + "new_count": len(new_ids), + "removed_count": len(removed_ids), + }) + + parts = [ + STICKY_COMMENT_MARKER, + "## Corpora Test Results", + "", + PREAMBLE, + "", + ] + if not rows: + parts += ["_0 findings on either side for the changed detectors._", ""] + return "\n".join(parts) + + parts += [build_top_line_summary(rows, changed), ""] + + if has_diff or any(r["is_new"] for r in rows): + rows.sort( + key=lambda r: ( + 0 if r["is_new"] else 1, + -(r["new_count"] + r["removed_count"]), + r["detector"], + ) + ) + else: + rows.sort(key=lambda r: r["detector"]) + + cols = ["Status", "Detector", "Unique matches (main)", "Unique matches (PR)", + "New", "Removed"] + aligns = ["", "", "---:", "---:", "---:", "---:"] + parts += [ + "| " + " | ".join(cols) + " |", + "|" + "|".join(a if a else "---" for a in aligns) + "|", + ] + + for r in rows: + if r["is_new"]: + cells = [ + r["emoji"], + r["detector"], + "—", + str(r["unique_pr"]), + "—", + "—", + ] + else: + cells = [ + r["emoji"], + r["detector"], + str(r["unique_main"]), + str(r["unique_pr"]), + str(r["new_count"]), + str(r["removed_count"]), + ] + parts.append("| " + " | ".join(cells) + " |") + parts.append("") + + no_data = [r["detector"] for r in rows if r["emoji"] == "⚪"] + if no_data: + names = ", ".join(f"`{d}`" for d in no_data) + subject = "this detector" if len(no_data) == 1 else "these detectors" + parts.append( + f"No comparison possible for: {names} (0 matches on main and PR; " + f"the corpus has no material for {subject})." + ) + parts.append("") + parts.append(STATUS_KEY) + parts.append("") + + return "\n".join(parts) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("main_jsonl") + parser.add_argument("pr_jsonl") + parser.add_argument("--changed-detectors", default="", + help="CSV of detectors changed in PR; filters report.") + parser.add_argument("--new-detectors", default="", + help="CSV of detectors present in PR but not main; rendered with 🆕.") + args = parser.parse_args() + + main_findings = load_findings(args.main_jsonl) + pr_findings = load_findings(args.pr_jsonl) + changed = parse_csv(args.changed_detectors) + new_detectors = parse_csv(args.new_detectors) + + sys.stdout.write(render( + main_findings, + pr_findings, + changed=changed if changed else None, + new_detectors=new_detectors, + )) + + +if __name__ == "__main__": + main()