From 9c6a5ad325af55f7a74355a62ad7a33fa7c057d9 Mon Sep 17 00:00:00 2001 From: dm-chelupati Date: Wed, 15 Jul 2026 09:50:42 -0700 Subject: [PATCH] Add azmon-alert-noise-filter-snow recipe and deploy script fixes Recipe: Azure Monitor alert noise filtering with ServiceNow MCP integration - Triage subagent with alert-noise-filter skill for transient detection, dependency deduplication, and repeat alert detection - PythonFunctionTool timer for observation window (time.sleep in subprocess) - ServiceNow MCP connector (stdio, @onlyflows/servicenow-mcp) for ticket creation - Response plan: Azure Monitor Sev0-3, merge window 1hr - Hooks: verify-alert-state-before-action, deny-prod-deletes, require-approval-for-restarts - Tool permissions: allow RunInTerminal/ExecutePythonCode, ask RunAzCliWriteCommands, deny FetchWebpage Deploy script fixes: - new-agent.sh: Add snowPassword -> SERVICENOW_PASSWORD and gitlabPat -> GITLAB_PAT secret mappings - apply-extras.sh: Add GitLab repo support (PAT auth via /api/v2/gitlab/auth), tool deployment via PUT /api/v1/extendedAgent/apply with Content-Type: application/yaml - assemble-agent.sh: Pass tools through Bicep params --- sreagent-templates/bicep/Apply-Extras.ps1 | 90 ++++++- sreagent-templates/bicep/apply-extras.sh | 234 ++++++++++++++---- sreagent-templates/bicep/assemble-agent.sh | 6 +- sreagent-templates/bin/new-agent.sh | 6 +- sreagent-templates/bin/ps/New-Agent.ps1 | 6 +- .../azmon-alert-noise-filter-snow/.gitignore | 0 .../azmon-alert-noise-filter-snow/README.md | 91 +++++++ .../azmon-alert-noise-filter-snow/agent.json | 109 ++++++++ .../azmon-all-severities.yaml | 16 ++ .../incident-platforms/azure-monitor.yaml | 5 + .../investigation-guidelines.yaml | 19 ++ .../config/common-prompts/safety-rules.yaml | 17 ++ .../config/github-domains/github-com.yaml | 7 + .../config/hooks/deny-prod-deletes.yaml | 11 + .../hooks/require-approval-for-restarts.yaml | 11 + .../config/repos/github-repo.yaml | 5 + .../config/skills/alert-noise-filter.md | 224 +++++++++++++++++ .../config/skills/alert-noise-filter.yaml | 15 ++ .../alert-triage-agent.instructions.md | 7 + .../config/subagents/alert-triage-agent.yaml | 20 ++ .../config/tools/wait-and-recheck-timer.yaml | 35 +++ .../connectors.json | 36 +++ .../data/incident-report-template.md | 65 +++++ .../expected-config.json | 49 ++++ .../tool-permissions.json | 13 + 25 files changed, 1034 insertions(+), 63 deletions(-) create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/.gitignore create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/README.md create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/agent.json create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-filters/azmon-all-severities.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-platforms/azure-monitor.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/investigation-guidelines.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/safety-rules.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/github-domains/github-com.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/deny-prod-deletes.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/require-approval-for-restarts.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/repos/github-repo.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.md create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.instructions.md create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/tools/wait-and-recheck-timer.yaml create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/connectors.json create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/data/incident-report-template.md create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/expected-config.json create mode 100644 sreagent-templates/recipes/azmon-alert-noise-filter-snow/tool-permissions.json diff --git a/sreagent-templates/bicep/Apply-Extras.ps1 b/sreagent-templates/bicep/Apply-Extras.ps1 index d6d7b6b5b..937f091ab 100644 --- a/sreagent-templates/bicep/Apply-Extras.ps1 +++ b/sreagent-templates/bicep/Apply-Extras.ps1 @@ -576,22 +576,26 @@ if ($ghDomainCount -gt 0) { # ═════════════════════════════════════════════════════════════════════════════ # 2. repos — data-plane only (requires azuresre.dev token) -# Split into byoapp_repos (domain has GitHubApp entry) and oauth_repos +# Split into byoapp_repos (domain has GitHubApp entry), gitlab_repos, and oauth_repos # ═════════════════════════════════════════════════════════════════════════════ $repos = $extras.repos $repoCount = ($repos | Measure-Object).Count $oauthRepos = @() $byoappRepos = @() +$gitlabRepos = @() if ($repoCount -gt 0) { if ($DpTokenAvailable) { foreach ($repo in $repos) { $rurl = $repo.spec.url + $rtype = if ($repo.spec.type) { $repo.spec.type } else { "" } if ($rurl -match '^http') { $rdomain = ([System.Uri]$rurl).Host } else { $rdomain = "github.com" } - if ($byoappDomains -contains $rdomain) { + if ($rdomain -like "*gitlab*" -or $rtype -ieq "GitLab") { + $gitlabRepos += $repo + } elseif ($byoappDomains -contains $rdomain) { $byoappRepos += $repo } else { $oauthRepos += $repo.name @@ -600,6 +604,9 @@ if ($repoCount -gt 0) { if ($byoappRepos.Count -gt 0) { Write-Host "repos: $($byoappRepos.Count) via BYO App (will be wired after githubDomains)" } + if ($gitlabRepos.Count -gt 0) { + Write-Host "repos: $($gitlabRepos.Count) via GitLab PAT (will be wired after gitlab/auth)" + } if ($oauthRepos.Count -gt 0) { Write-Host "repos: $($oauthRepos.Count) via OAuth (will be wired after GitHub sign-in below)" } @@ -886,18 +893,43 @@ if ($saCount -gt 0) { } # ═════════════════════════════════════════════════════════════════════════════ -# 4g-3. tools — data-plane PUT -# Route: PUT /api/v2/extendedAgent/tools/{name} +# 4g-3. tools — data-plane POST via /api/v1/extendedAgent/apply (ToolList kind) +# Route: POST /api/v1/extendedAgent/apply # ═════════════════════════════════════════════════════════════════════════════ $toolItems = if ($extras.PSObject.Properties['tools']) { $extras.tools } else { $null } $tlCount = ($toolItems | Measure-Object).Count if ($tlCount -gt 0) { if ($DpTokenAvailable) { Write-Host "tools: $tlCount" - foreach ($tl in $toolItems) { - $name = if ($tl.metadata) { $tl.metadata.name } else { $tl.name } - $props = if ($tl.spec) { $tl.spec } else { $tl.properties } - DataPlane-PutExtended -Kind "tools" -Name $name -Type "Tool" -Tags @() -Properties $props + $token = Get-DpToken + # Build ToolList YAML document + $toolList = @{ + api_version = "azuresre.ai/v1" + kind = "ToolList" + metadata = @{} + spec = @{ + tools = @(foreach ($tl in $toolItems) { + $name = if ($tl.metadata) { $tl.metadata.name } else { $tl.name } + $spec = if ($tl.spec) { $tl.spec } else { $tl.properties } + @{ metadata = @{ name = $name }; spec = $spec } + }) + } + } + # Convert to YAML-like format (use JSON as fallback, apply endpoint accepts both) + $toolsBody = $toolList | ConvertTo-Json -Depth 20 + try { + $null = Invoke-RestMethod -Uri "$AgentEndpoint/api/v1/extendedAgent/apply" ` + -Method Put -Headers @{ Authorization = "Bearer $token"; "Content-Type" = "application/json" } ` + -Body $toolsBody -ErrorAction Stop + Write-Host " ok tools ($tlCount via apply endpoint)" + } catch { + Write-Host " FAILED — PUT /api/v1/extendedAgent/apply (ToolList)" + # Fallback: try individual v2 PUT + foreach ($tl in $toolItems) { + $name = if ($tl.metadata) { $tl.metadata.name } else { $tl.name } + $props = if ($tl.spec) { $tl.spec } else { $tl.properties } + DataPlane-PutExtended -Kind "tools" -Name $name -Type "Tool" -Tags @() -Properties $props + } } } else { Write-Host "tools: $tlCount - WARNING skipped (no data-plane token)" @@ -1153,6 +1185,48 @@ if ($DpTokenAvailable) { } } + # ───────────────────────────────────────────────────────────────────────── + # GitLab repos — authenticate via PAT then push repos + # ───────────────────────────────────────────────────────────────────────── + if ($gitlabRepos.Count -gt 0) { + $gitlabPat = $env:GITLAB_PAT + if ($gitlabPat) { + Write-Host "gitlab repos: $($gitlabRepos.Count)" + $token = Get-DpToken + # Store GitLab PAT + try { + $null = Invoke-RestMethod -Uri "$AgentEndpoint/api/v2/gitlab/auth" ` + -Method Put -Headers @{ Authorization = "Bearer $token"; "Content-Type" = "application/json" } ` + -Body (@{ pat = $gitlabPat } | ConvertTo-Json -Compress) -ErrorAction Stop + Write-Host " ok gitlab/auth (PAT stored)" + } catch { + Write-Host " FAILED - PUT /api/v2/gitlab/auth" + } + # Push repos + foreach ($repo in $gitlabRepos) { + $rname = $repo.name + $rurl = $repo.spec.url + if ($rurl -notmatch '^http' -and $rurl -match '/') { + $rurl = "https://gitlab.com/$rurl" + } + $rdesc = if ($repo.spec.description) { $repo.spec.description } else { "" } + $rbody = @{ name = $rname; type = "CodeRepo"; properties = @{ url = $rurl; type = "GitLab" } } + if ($rdesc) { $rbody.properties.description = $rdesc } + $rbodyJson = $rbody | ConvertTo-Json -Depth 5 -Compress + try { + $null = Invoke-RestMethod -Uri "$AgentEndpoint/api/v2/repos/$([Uri]::EscapeDataString($rname))" ` + -Method Put -Headers @{ Authorization = "Bearer $token"; "Content-Type" = "application/json" } ` + -Body $rbodyJson -ErrorAction Stop + Write-Host " ok repo/$rname ($rurl) [GitLab]" + } catch { + Write-Host " FAILED - PUT /api/v2/repos/$rname (try the portal Repos blade)" + } + } + } else { + Write-Host "gitlab repos: $($gitlabRepos.Count) - WARNING skipped (GITLAB_PAT not set)" + } + } + # ───────────────────────────────────────────────────────────────────────── # GitHub: OAuth sign-in + connector + repo wiring. # ───────────────────────────────────────────────────────────────────────── diff --git a/sreagent-templates/bicep/apply-extras.sh b/sreagent-templates/bicep/apply-extras.sh index 629ed1f19..178ed9aa9 100755 --- a/sreagent-templates/bicep/apply-extras.sh +++ b/sreagent-templates/bicep/apply-extras.sh @@ -413,12 +413,14 @@ if [[ "$count" -gt 0 ]]; then fi # 2. repos — data-plane only (requires azuresre.dev token) -# Split repos into two buckets: -# - byoapp_repos: domain has a GitHubApp entry in githubDomains → push directly after githubDomains are applied -# - oauth_repos: domain uses OAuth/PAT → pushed in the OAuth sign-in block (step 5) +# Split repos into three buckets: +# - byoapp_repos: domain has a GitHubApp entry in githubDomains → push directly after githubDomains are applied +# - gitlab_repos: domain is gitlab.com → push directly (auth via /api/v2/gitlab/auth PAT) +# - oauth_repos: domain uses OAuth/PAT → pushed in the OAuth sign-in block (step 5) count=$(jq '[.repos // [] | .[] | select(.spec.url // "" | length > 0)] | length' "$FILE") oauth_repos=() byoapp_repos=() +gitlab_repos=() # Build a set of domains that use GitHubApp auth (BYO App) _byoapp_domains=$(jq -r '[.githubDomains // [] | .[] | select(.spec.authType == "GitHubApp") | .metadata.name // .name] | join("|")' "$FILE" 2>/dev/null) if [[ "$count" -gt 0 ]]; then @@ -426,13 +428,16 @@ if [[ "$count" -gt 0 ]]; then for i in $(seq 0 $((count - 1))); do name=$(jq -r --argjson i "$i" '[.repos[] | select(.spec.url // "" | length > 0)][$i].name' "$FILE") rurl=$(jq -r --argjson i "$i" '[.repos[] | select(.spec.url // "" | length > 0)][$i].spec.url' "$FILE") + rtype=$(jq -r --argjson i "$i" '[.repos[] | select(.spec.url // "" | length > 0)][$i].spec.type // ""' "$FILE") # Determine the domain: full URL → extract host; short "org/repo" → github.com if [[ "$rurl" == http* ]]; then rdomain=$(echo "$rurl" | sed 's|https\?://||' | cut -d/ -f1) else rdomain="github.com" fi - if [[ -n "$_byoapp_domains" ]] && echo "$rdomain" | grep -qE "^(${_byoapp_domains})$"; then + if [[ "$rdomain" == *"gitlab"* || "$(echo "$rtype" | tr '[:upper:]' '[:lower:]')" == "gitlab" ]]; then + gitlab_repos+=("$name") + elif [[ -n "$_byoapp_domains" ]] && echo "$rdomain" | grep -qE "^(${_byoapp_domains})$"; then byoapp_repos+=("$name") else oauth_repos+=("$name") @@ -441,6 +446,9 @@ if [[ "$count" -gt 0 ]]; then if [[ ${#byoapp_repos[@]} -gt 0 ]]; then echo "repos: ${#byoapp_repos[@]} via BYO App (will be wired after githubDomains)" fi + if [[ ${#gitlab_repos[@]} -gt 0 ]]; then + echo "repos: ${#gitlab_repos[@]} via GitLab PAT (will be wired after gitlab/auth)" + fi if [[ ${#oauth_repos[@]} -gt 0 ]]; then echo "repos: ${#oauth_repos[@]} via OAuth (will be wired after GitHub sign-in below)" fi @@ -737,11 +745,15 @@ if [[ ${#byoapp_repos[@]} -gt 0 && "$DP_TOKEN_AVAILABLE" == "true" ]]; then rtype_in=$(jq -r --arg n "$rname" '[.repos[] | select(.name == $n)][0].spec.type // "github"' "$FILE") case "$(printf %s "$rtype_in" | tr "[:upper:]" "[:lower:]")" in ado|azuredevops|azure-devops) rtype="AzureDevOps" ;; + gitlab|gl) rtype="GitLab" ;; *) rtype="GitHub" ;; esac # Normalize short "org/repo" to full URL if [[ "$rurl" != http* && "$rurl" == */* ]]; then - rurl="https://github.com/${rurl}" + case "$rtype" in + GitLab) rurl="https://gitlab.com/${rurl}" ;; + *) rurl="https://github.com/${rurl}" ;; + esac fi rbody=$(jq -nc --arg n "$rname" --arg u "$rurl" --arg t "$rtype" --arg d "$rdesc" '{ name: $n, @@ -759,6 +771,57 @@ if [[ ${#byoapp_repos[@]} -gt 0 && "$DP_TOKEN_AVAILABLE" == "true" ]]; then done fi +# 4f-3c. GitLab native connector — data-plane PUT /api/v2/gitlab/auth +# GitLab uses a dedicated auth endpoint (not ConnectorV2, not a toggle). +# Stores PAT in agent, validates against gitlab.com/api/v4/user. +if [[ -n "${GITLAB_PAT:-}" ]]; then + if [[ "$DP_TOKEN_AVAILABLE" == "true" ]]; then + echo "gitlab: configuring native connector (PAT)" + TOKEN=$(_dp_token) + gl_result=$(curl -sS -w "\n%{http_code}" -X PUT \ + "${AGENT_ENDPOINT}/api/v2/gitlab/auth" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data "{\"Pat\":\"${GITLAB_PAT}\"}" 2>&1) + gl_code=$(echo "$gl_result" | tail -1) + if [[ "$gl_code" =~ ^2 ]]; then + echo " ok gitlab/auth (PAT stored and validated)" + else + gl_err=$(echo "$gl_result" | sed '$d' | head -2) + echo " FAILED — PUT /api/v2/gitlab/auth (HTTP ${gl_code})" + echo " ${gl_err}" + echo " Verify your GITLAB_PAT has read_api + read_repository scopes" + fi + + # Wire GitLab repos (PAT already stored above) + if [[ ${#gitlab_repos[@]} -gt 0 ]]; then + echo "gitlab repos: ${#gitlab_repos[@]}" + TOKEN=$(_dp_token) + repo_count=$(jq '.repos // [] | length' "$FILE") + for rname in "${gitlab_repos[@]}"; do + rurl=$(jq -r --arg n "$rname" '[.repos[] | select(.name == $n)][0].spec.url' "$FILE") + rdesc=$(jq -r --arg n "$rname" '[.repos[] | select(.name == $n)][0].spec.description // ""' "$FILE") + rbody=$(jq -nc --arg n "$rname" --arg u "$rurl" --arg d "$rdesc" '{ + name: $n, + type: "CodeRepo", + properties: ({ url: $u, type: "GitLab" } + (if $d == "" then {} else { description: $d } end)) + }') + if curl -sS -f -X PUT "${AGENT_ENDPOINT}/api/v2/repos/$(printf %s "$rname" | jq -sRr @uri)" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data "$rbody" >/dev/null 2>&1; then + echo " ok repo/${rname} (${rurl}) [GitLab PAT]" + else + echo " FAILED — PUT /api/v2/repos/${rname}" + fi + done + fi + else + echo "gitlab: ⚠ skipped (no data-plane token). Set GITLAB_PAT and re-run." + DP_SKIPPED_ITEMS+=("gitlab/auth") + fi +fi + # 4f-4. connectorV2 — data-plane multi-step setup via /api/v2/connectorV2 # Each entry: { metadata: { name }, spec: { apiName, displayName, connectionName?, # parameterValueSet?: { name, values }, requireApprovalTools?: [...] } } @@ -776,48 +839,99 @@ if [[ "$count" -gt 0 ]]; then cv2_pvs=$(echo "$cv2_spec" | jq -c '.parameterValueSet // null') cv2_pv=$(echo "$cv2_spec" | jq -c '.parameterValues // null') cv2_rat=$(echo "$cv2_spec" | jq -c '.requireApprovalTools // null') + cv2_skip=$(echo "$cv2_spec" | jq -r '.skipConnection // false') TOKEN=$(_dp_token) - # Step 1: Create the connection - conn_body=$(jq -nc --arg dn "$cv2_display" --arg cn "$cv2_api" \ - --argjson pvs "$cv2_pvs" --argjson pv "$cv2_pv" \ - '{displayName: $dn, connectorName: $cn} + (if $pvs != null then {parameterValueSet: $pvs} else {} end) + (if $pv != null then {parameterValues: $pv} else {} end)') - conn_result=$(curl -sS -w "\n%{http_code}" -X PUT \ - "${AGENT_ENDPOINT}/api/v2/connectorV2/connections/${cv2_conn}" \ - -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - --data "$conn_body" 2>&1) - conn_code=$(echo "$conn_result" | tail -1) - if [[ "$conn_code" =~ ^2 ]]; then - echo " ok connectorV2/connection/${cv2_conn}" + # Step 1: Create the connection (skip if connector uses dedicated auth endpoint) + if [[ "$cv2_skip" == "true" ]]; then + echo " skip connectorV2/connection/${cv2_conn} (uses dedicated auth endpoint)" else - echo " WARN — PUT connection/${cv2_conn} (HTTP ${conn_code}) — may need OAuth consent in portal" + conn_body=$(jq -nc --arg dn "$cv2_display" --arg cn "$cv2_api" \ + --argjson pvs "$cv2_pvs" --argjson pv "$cv2_pv" \ + '{displayName: $dn, connectorName: $cn} + (if $pvs != null then {parameterValueSet: $pvs} else {} end) + (if $pv != null then {parameterValues: $pv} else {} end)') + conn_result=$(curl -sS -w "\n%{http_code}" -X PUT \ + "${AGENT_ENDPOINT}/api/v2/connectorV2/connections/${cv2_conn}" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data "$conn_body" 2>&1) + conn_code=$(echo "$conn_result" | tail -1) + if [[ "$conn_code" =~ ^2 ]]; then + echo " ok connectorV2/connection/${cv2_conn}" + else + echo " WARN — PUT connection/${cv2_conn} (HTTP ${conn_code}) — may need OAuth consent in portal" + fi fi - # Step 2: Create MCP server config (links connection to MCP tools) - mcp_body=$(jq -nc --arg desc "$cv2_display" --arg cn "$cv2_conn" --arg api "$cv2_api" \ - --argjson rat "$cv2_rat" \ - '{properties: {description: $desc, connectors: [{name: $api, connectionName: $cn}]}} + (if $rat != null then {runtimeMcpConfiguration: {requireApprovalTools: $rat}} else {} end)') - TOKEN=$(_dp_token) - mcp_result=$(curl -sS -w "\n%{http_code}" -X PUT \ - "${AGENT_ENDPOINT}/api/v2/connectorV2/mcpservers/${cv2_conn}" \ - -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - --data "$mcp_body" 2>&1) - mcp_code=$(echo "$mcp_result" | tail -1) - if [[ "$mcp_code" =~ ^2 ]]; then - echo " ok connectorV2/mcpserver/${cv2_conn}" + # Step 2: Create MCP server config (skip if connection was skipped) + if [[ "$cv2_skip" == "true" ]]; then + echo " skip connectorV2/mcpserver/${cv2_conn} (uses dedicated auth endpoint)" else - echo " FAILED — PUT mcpservers/${cv2_conn} (HTTP ${mcp_code})" - echo " $(echo "$mcp_result" | sed '$d' | head -2)" + mcp_body=$(jq -nc --arg desc "$cv2_display" --arg cn "$cv2_conn" --arg api "$cv2_api" \ + --argjson rat "$cv2_rat" \ + '{properties: {description: $desc, connectors: [{name: $api, connectionName: $cn}]}} + (if $rat != null then {runtimeMcpConfiguration: {requireApprovalTools: $rat}} else {} end)') + TOKEN=$(_dp_token) + mcp_result=$(curl -sS -w "\n%{http_code}" -X PUT \ + "${AGENT_ENDPOINT}/api/v2/connectorV2/mcpservers/${cv2_conn}" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data "$mcp_body" 2>&1) + mcp_code=$(echo "$mcp_result" | tail -1) + if [[ "$mcp_code" =~ ^2 ]]; then + echo " ok connectorV2/mcpserver/${cv2_conn}" + else + echo " FAILED — PUT mcpservers/${cv2_conn} (HTTP ${mcp_code})" + echo " $(echo "$mcp_result" | sed '$d' | head -2)" + fi fi - # Step 3: Print consent link if connection needs OAuth - conn_status=$(echo "$conn_result" | sed '$d' | jq -r '.properties.overallStatus // "Unknown"' 2>/dev/null) - if [[ "$conn_status" == "Error" || "$conn_status" == "Unauthenticated" ]]; then - echo " ⚠ Connection ${cv2_conn} needs OAuth consent. Complete in the portal:" - echo " https://sre.azure.com → Connectors → ${cv2_display} → Authorize" + # Step 3: For PAT-based connectors, store credentials via dedicated auth endpoint + cv2_auth_type=$(echo "$cv2_spec" | jq -r '.authType // empty' | tr '[:upper:]' '[:lower:]') + if [[ "$cv2_api" == "gitlab" && "$cv2_auth_type" == "pat" ]]; then + # Read PAT from GITLAB_PAT env var (sourced from connectors.secrets.env) + cv2_pat="${GITLAB_PAT:-}" + if [[ -n "$cv2_pat" ]]; then + TOKEN=$(_dp_token) + gl_auth_result=$(curl -sS -w "\n%{http_code}" -X PUT \ + "${AGENT_ENDPOINT}/api/v2/gitlab/auth" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data "{\"Pat\":\"${cv2_pat}\"}" 2>&1) + gl_auth_code=$(echo "$gl_auth_result" | tail -1) + if [[ "$gl_auth_code" =~ ^2 ]]; then + echo " ok gitlab/auth (PAT stored)" + else + echo " WARN — PUT gitlab/auth (HTTP ${gl_auth_code}) — verify PAT is valid" + fi + else + echo " ⚠ GitLab PAT not resolved — set GITLAB_PAT env var or pass via connectors.secrets.env" + fi + fi + + # Step 4: Get consent URL (only for OAuth connectors that created a connection) + if [[ "$cv2_skip" != "true" ]]; then + conn_status=$(echo "$conn_result" | sed '$d' | jq -r '.properties.overallStatus // "Unknown"' 2>/dev/null) + if [[ "$conn_status" == "Error" || "$conn_status" == "Unauthenticated" ]]; then + # Try to get consent URL from the connection response + cv2_consent_url=$(echo "$conn_result" | sed '$d' | jq -r '.properties.consentUrl // .properties.connectionParameters.token.oAuthSettings.redirectUrl // empty' 2>/dev/null) + # If not in response, try the consentLinks endpoint + if [[ -z "$cv2_consent_url" ]]; then + TOKEN=$(_dp_token) + cv2_consent_resp=$(curl -sS -f -X POST \ + "${AGENT_ENDPOINT}/api/v2/connectorV2/connections/${cv2_conn}/consentLinks" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + --data '{"parameters":[{"parameterName":"token","redirectUrl":"https://sre.azure.com"}]}' 2>/dev/null || echo '{}') + cv2_consent_url=$(echo "$cv2_consent_resp" | jq -r '.value[0].link // .link // .consentUrl // empty' 2>/dev/null) + fi + if [[ -n "$cv2_consent_url" ]]; then + echo " ⚠ Connection ${cv2_conn} needs OAuth. Open this URL to authorize:" + echo " ${cv2_consent_url}" + else + echo " ⚠ Connection ${cv2_conn} needs OAuth consent. Complete in the portal:" + echo " https://sre.azure.com → Connectors → ${cv2_display} → Authorize" + fi + fi fi done else @@ -942,18 +1056,38 @@ if [[ "$count" -gt 0 ]]; then fi fi -# 4i-3. tools — data-plane PUT -# Route: PUT /api/v2/extendedAgent/tools/{name} -# Body: { name, type: "Tool", tags: [], properties: { ... tool spec } } +# 4i-3. tools — data-plane POST via /api/v1/extendedAgent/apply (ToolList kind) +# Route: POST /api/v1/extendedAgent/apply +# Body: { api_version: "azuresre.ai/v1", kind: "ToolList", metadata: {}, spec: { tools: [...] } } count=$(jq '.tools // [] | length' "$FILE") if [[ "$count" -gt 0 ]]; then if [[ "$DP_TOKEN_AVAILABLE" == "true" ]]; then echo "tools: ${count}" - for i in $(seq 0 $((count - 1))); do - name=$(jq -r --argjson i "$i" '.tools[$i].metadata.name' "$FILE") - props=$(jq -c --argjson i "$i" '.tools[$i].spec' "$FILE") - dataplane_put_extended "tools" "$name" "Tool" "[]" "$props" - done + TOKEN=$(_dp_token) + # Build YAML payload from the JSON tools array + tools_yaml=$(python3 -c " +import json, sys, yaml +data = json.load(open(sys.argv[1])) +tools = [t['spec'] for t in data.get('tools', [])] +doc = {'api_version': 'azuresre.ai/v1', 'kind': 'ToolList', 'metadata': {}, 'spec': {'tools': tools}} +print(yaml.dump(doc, default_flow_style=False)) +" "$FILE" 2>/dev/null) + if [[ -z "$tools_yaml" ]]; then + echo " FAILED — could not generate YAML (is PyYAML installed?)" + else + apply_result=$(curl -sS -w "\n%{http_code}" -X PUT \ + "${AGENT_ENDPOINT}/api/v1/extendedAgent/apply" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/yaml" \ + --data "$tools_yaml" 2>&1) + apply_code=$(echo "$apply_result" | tail -1) + if [[ "$apply_code" =~ ^2 ]]; then + echo " ok tools (${count} deployed via apply)" + else + echo " FAILED — PUT tools/apply (HTTP ${apply_code})" + echo " $(echo "$apply_result" | sed '$d' | head -2)" + fi + fi else echo "tools: ${count} — ⚠ skipped (no data-plane token)" for i in $(seq 0 $((count - 1))); do @@ -1128,12 +1262,16 @@ if [[ ${#oauth_repos[@]} -gt 0 ]]; then rurl=$(jq -r --argjson i "$i" '.repos[$i].spec.url' "$FILE") # Normalize short "org/repo" to full URL (API requires valid URL format) if [[ "$rurl" != http* && "$rurl" == */* ]]; then - rurl="https://github.com/${rurl}" + case "$(printf %s "$rtype_in" | tr "[:upper:]" "[:lower:]")" in + gitlab|gl) rurl="https://gitlab.com/${rurl}" ;; + *) rurl="https://github.com/${rurl}" ;; + esac fi - # Map our spec.type ("github"/"ado") to the View enum ("GitHub"/"AzureDevOps"). + # Map our spec.type ("github"/"ado"/"gitlab") to the View enum. rtype_in=$(jq -r --argjson i "$i" '.repos[$i].spec.type // "github"' "$FILE") case "$(printf %s "$rtype_in" | tr "[:upper:]" "[:lower:]")" in ado|azuredevops|azure-devops) rtype="AzureDevOps" ;; + gitlab|gl) rtype="GitLab" ;; *) rtype="GitHub" ;; esac rdesc=$(jq -r --argjson i "$i" '.repos[$i].spec.description // ""' "$FILE") diff --git a/sreagent-templates/bicep/assemble-agent.sh b/sreagent-templates/bicep/assemble-agent.sh index 3179dccd5..fe66949e1 100755 --- a/sreagent-templates/bicep/assemble-agent.sh +++ b/sreagent-templates/bicep/assemble-agent.sh @@ -271,8 +271,8 @@ fi GITHUB_DOMAINS=$(resolve_env_vars "$(collect_config "github-domains")") _log "github-domains: $(echo "$GITHUB_DOMAINS" | jq 'length')" -# ConnectorV2 (Jira, Slack, etc.) — config/connectorv2/*.yaml -CONNECTORV2=$(collect_config "connectorv2") +# ConnectorV2 (Jira, Slack, GitLab, etc.) — config/connectorv2/*.yaml +CONNECTORV2=$(resolve_env_vars "$(collect_config "connectorv2")") _log "connectorv2: $(echo "$CONNECTORV2" | jq 'length')" MARKETPLACES="[]" @@ -388,7 +388,7 @@ jq -n \ "enableWebhookBridge": { "value": ($toggles.enableWebhookBridge // false) }, "webhookBridgeTriggerUrl": { "value": ($toggles.webhookBridgeTriggerUrl // "") }, "connectors": { "value": [$connectors[] | select(.properties.dataConnectorType != "KnowledgeFile")] }, - "tools": { "value": [] }, + "tools": { "value": $tools }, "skills": { "value": [] }, "subagents": { "value": [] }, "scheduledTasks": { "value": [] }, diff --git a/sreagent-templates/bin/new-agent.sh b/sreagent-templates/bin/new-agent.sh index a960cfb8d..388f1f08b 100755 --- a/sreagent-templates/bin/new-agent.sh +++ b/sreagent-templates/bin/new-agent.sh @@ -264,8 +264,10 @@ while IFS="=" read -r key _val || [[ -n "$key" ]]; do env_name=$(echo "$key" | sed 's/[a-z]/\U&/g;s/[^A-Z0-9]/_/g') # Map known secrets to their connector env var names case "$key" in - dtToken) echo "DYNATRACE_BEARER_TOKEN=$(_get "$VALUES_FILE" "$key")" >> "$SECRETS_ENV" ;; - *) echo "${env_name}=$(_get "$VALUES_FILE" "$key")" >> "$SECRETS_ENV" ;; + dtToken) echo "DYNATRACE_BEARER_TOKEN=$(_get "$VALUES_FILE" "$key")" >> "$SECRETS_ENV" ;; + snowPassword) echo "SERVICENOW_PASSWORD=$(_get "$VALUES_FILE" "$key")" >> "$SECRETS_ENV" ;; + gitlabPat) echo "GITLAB_PAT=$(_get "$VALUES_FILE" "$key")" >> "$SECRETS_ENV" ;; + *) echo "${env_name}=$(_get "$VALUES_FILE" "$key")" >> "$SECRETS_ENV" ;; esac fi done < "$VALUES_FILE" diff --git a/sreagent-templates/bin/ps/New-Agent.ps1 b/sreagent-templates/bin/ps/New-Agent.ps1 index ebb3bd464..362507300 100644 --- a/sreagent-templates/bin/ps/New-Agent.ps1 +++ b/sreagent-templates/bin/ps/New-Agent.ps1 @@ -322,8 +322,10 @@ foreach ($kvp in $Values.GetEnumerator()) { $isSecret = if ($Prompts.PSObject.Properties[$k] -and $Prompts.$k.PSObject.Properties['secret'] -and $Prompts.$k.secret -eq $true) { $true } else { $false } if ($isSecret -and -not [string]::IsNullOrEmpty($v)) { $envName = switch ($k) { - "dtToken" { "DYNATRACE_BEARER_TOKEN" } - default { ($k -creplace '[a-z]', { $_.Value.ToUpper() }) -replace '[^A-Z0-9]', '_' } + "dtToken" { "DYNATRACE_BEARER_TOKEN" } + "snowPassword" { "SERVICENOW_PASSWORD" } + "gitlabPat" { "GITLAB_PAT" } + default { ($k -creplace '[a-z]', { $_.Value.ToUpper() }) -replace '[^A-Z0-9]', '_' } } $secretLines += "$envName=$v" } diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/.gitignore b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/README.md b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/README.md new file mode 100644 index 000000000..03777b0a3 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/README.md @@ -0,0 +1,91 @@ +# Azure Monitor Alert Noise Filter + ServiceNow Ticket Recipe + +## Overview + +This recipe reduces alert noise by filtering transient Azure Monitor alerts before they become ServiceNow tickets: + +1. **Azure Monitor alert fires** → routed to the **Alert Triage Agent** +2. **15-minute transient check** — the triage agent runs Python code to wait 15 minutes, then re-checks whether the alert is still in Fired state +3. **Transient alert** (resolved within 15 min) → closed automatically with a summary. No ticket created. +4. **Persistent alert** (still firing after 15 min) → escalated to the **Alert Investigator** for deep analysis +5. **Investigation** — correlates App Insights, Log Analytics, Azure Monitor metrics, and Activity Logs to find root cause +6. **ServiceNow ticket** — creates an incident in ServiceNow via MCP with full investigation details + +## Architecture + +``` +Azure Monitor Alert (Fired) + │ + ▼ +┌─────────────────────┐ +│ Alert Triage Agent │ +│ (15-min timer) │ +└────────┬────────────┘ + │ + ┌────┴────┐ + │ │ +Resolved Still Fired + │ │ + ▼ ▼ + Close ┌──────────────────┐ + Alert │ Alert Investigator │ + │ (deep analysis) │ + └────────┬───────────┘ + │ + ▼ + ┌─────────────────┐ + │ ServiceNow MCP │ + │ (create ticket) │ + └─────────────────┘ +``` + +## Data Sources + +| Source | Purpose | +|---|---| +| Azure Monitor | Alert ingestion (incident platform), metric queries | +| Application Insights | Exception traces, failed requests, dependencies | +| Log Analytics | Correlated logs, custom queries | +| ServiceNow (MCP connector) | Incident ticket creation/update via `sn_mcp` server | + +## Prerequisites + +- Azure subscription with resource groups to monitor +- Application Insights and/or Log Analytics workspace (recommended) +- ServiceNow instance with MCP server enabled (`sn_mcp` plugin) +- ServiceNow user with incident create/update permissions + +## Quick Start + +```bash +# Set required environment variables +azd env set RECIPE azmon-servicenow-transient-check +azd env set AZURE_AGENT_NAME my-alert-triage-agent +azd env set AZURE_RESOURCE_GROUP sre-agent-rg +azd env set AZURE_LOCATION swedencentral +azd env set AZURE_TARGET_RGS "rg-app-prod,rg-db-prod" + +# Optional: App Insights / LAW +azd env set AZURE_AI_ID /subscriptions/.../applicationInsights/my-ai +azd env set AZURE_AI_APPID +azd env set AZURE_LAW_ID /subscriptions/.../workspaces/my-law + +# Deploy +azd up +``` + +Then configure the ServiceNow MCP connector in the portal with your instance credentials. + +## Subagents + +| Agent | Role | +|---|---| +| `alert-triage-agent` | First responder. Runs 15-min transient check. Routes alerts. | +| `alert-investigator` | Deep investigation. Creates ServiceNow tickets. | + +## Skills + +| Skill | Purpose | +|---|---| +| `transient-alert-checker` | Python timer + alert state re-check after 15 minutes | +| `investigate-persistent-alert` | Full investigation + ServiceNow ticket creation | diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/agent.json b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/agent.json new file mode 100644 index 000000000..6910bd050 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/agent.json @@ -0,0 +1,109 @@ +{ + "_scenario": "azmon-alert-noise-filter-snow", + "_description": "SRE Agent that filters noisy Azure Monitor alerts using a 15-minute transient check — auto-closes self-resolving alerts and only creates ServiceNow tickets (via MCP) for real, persistent issues.", + "_prerequisites": [ + "Azure subscription with at least one resource group to monitor", + "Application Insights and/or Log Analytics workspace", + "ServiceNow instance with MCP server enabled and API access" + ], + "_prompts": { + "agentName": { + "ask": "Agent name (lowercase, hyphens ok)", + "default": "my-sre-agent" + }, + "resourceGroup": { + "ask": "Resource group for the agent", + "default": "sre-agent-rg" + }, + "location": { + "ask": "Region", + "options": [ + "eastus2", + "swedencentral", + "uksouth", + "australiaeast" + ], + "required": true + }, + "targetRGs": { + "ask": "Resource groups to monitor (comma-separated)", + "required": true + }, + "appInsightsId": { + "ask": "Application Insights resource ID (or leave blank)", + "default": "" + }, + "appInsightsAppId": { + "ask": "Application Insights App ID / GUID (or leave blank)", + "default": "" + }, + "lawId": { + "ask": "Log Analytics workspace resource ID (or leave blank)", + "default": "" + }, + "snowInstance": { + "ask": "ServiceNow instance name (e.g. dev181595 — the subdomain in https://dev181595.service-now.com)", + "required": true, + "default": "dev181595" + }, + "snowUser": { + "ask": "ServiceNow username", + "required": true, + "default": "aes.creator" + }, + "snowPassword": { + "ask": "ServiceNow password", + "required": true, + "secret": true + }, + "existingUamiId": { + "ask": "Existing UAMI resource ID (leave blank to create new)", + "default": "" + }, + "githubRepo": { + "ask": "GitHub repo URL (or leave blank)", + "default": "" + }, + "modelProvider": { + "ask": "AI model provider", + "options": [ + "Anthropic", + "Azure OpenAI" + ], + "default": "Anthropic" + }, + "existingAgentAppInsightsId": { + "ask": "Existing App Insights resource ID for agent telemetry (leave blank to create new)", + "default": "" + }, + "githubAppClientId": { + "ask": "GitHub App Client ID for BYO App auth (starts with Iv..., leave blank for OAuth)", + "default": "" + }, + "privateKeySecretUri": { + "ask": "Key Vault key URI for GitHub App private key (https://.vault.azure.net/keys/)", + "default": "" + } + }, + "identity": { + "agentName": "{{agentName}}", + "resourceGroup": "{{resourceGroup}}", + "subscription": "", + "location": "{{location}}", + "targetResourceGroups": "{{targetRGs}}" + }, + "access": { + "accessLevel": "High", + "actionMode": "Review" + }, + "upgradeChannel": "Preview", + "defaultModelProvider": "{{modelProvider}}", + "monthlyAgentUnitLimit": 10000, + "tags": {}, + "toggles": { + "enableWebhookBridge": false, + "webhookBridgeTriggerUrl": "" + }, + "existingUamiId": "{{existingUamiId}}", + "existingAgentAppInsightsId": "{{existingAgentAppInsightsId}}" +} diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-filters/azmon-all-severities.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-filters/azmon-all-severities.yaml new file mode 100644 index 000000000..c2e2b5161 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-filters/azmon-all-severities.yaml @@ -0,0 +1,16 @@ +metadata: + name: azmon-all-severities +spec: + incidentPlatform: AzMonitor + isEnabled: true + priorities: + - Sev0 + - Sev1 + - Sev2 + - Sev3 + handlingAgent: alert-triage-agent + agentMode: Autonomous + deepInvestigationEnabled: false + maxAutomatedInvestigationAttempts: 3 + mergeEnabled: true + mergeWindowHours: 1 diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-platforms/azure-monitor.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-platforms/azure-monitor.yaml new file mode 100644 index 000000000..41e3a7c49 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/automations/incident-platforms/azure-monitor.yaml @@ -0,0 +1,5 @@ +name: azure-monitor +spec: + platformType: AzMonitor + displayName: Azure Monitor Alerts + description: Receives Azure Monitor fired alerts as incidents diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/investigation-guidelines.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/investigation-guidelines.yaml new file mode 100644 index 000000000..0dcefb00e --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/investigation-guidelines.yaml @@ -0,0 +1,19 @@ +metadata: + name: investigation-guidelines +spec: + prompt: '## Investigation guidelines + + + - Always check the last 3 deployments for correlation + + - Include timestamp, affected resource, and severity in all summaries + + - Never take destructive actions without explicit approval + + - Prefer read-only investigation before recommending changes + + - Always provide an impact assessment (users affected, blast radius) + + - For transient alerts: wait the full 15-minute timer before concluding + + - Always include alert state transition history in ServiceNow tickets' diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/safety-rules.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/safety-rules.yaml new file mode 100644 index 000000000..9bc21ebed --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/common-prompts/safety-rules.yaml @@ -0,0 +1,17 @@ +metadata: + name: safety-rules +spec: + prompt: '## Safety rules + + + - Never delete resources in production without explicit approval + + - Always prefer read-only investigation before taking action + + - Escalate to human if confidence is below 80% + + - Do not modify network security groups or firewall rules + + - Do not access or display secrets, keys, or connection strings + + - Always wait for the 15-minute transient check to complete before escalating' diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/github-domains/github-com.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/github-domains/github-com.yaml new file mode 100644 index 000000000..041c26b9f --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/github-domains/github-com.yaml @@ -0,0 +1,7 @@ +metadata: + name: github.com +spec: + authType: GitHubApp + clientId: "{{githubAppClientId}}" + privateKeySecretUri: "{{privateKeySecretUri}}" + keyVaultManagedIdentityId: "{{existingUamiId}}" diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/deny-prod-deletes.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/deny-prod-deletes.yaml new file mode 100644 index 000000000..4545f0aae --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/deny-prod-deletes.yaml @@ -0,0 +1,11 @@ +metadata: + name: deny-prod-deletes +spec: + eventType: PreToolUse + hook: + type: prompt + prompt: If the tool targets a production resource (name contains 'prod' or 'prd'), + deny the action. Otherwise allow. + matcher: ^(delete_|remove_).* + permissionDecision: deny + enabled: true diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/require-approval-for-restarts.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/require-approval-for-restarts.yaml new file mode 100644 index 000000000..3eae406c9 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/hooks/require-approval-for-restarts.yaml @@ -0,0 +1,11 @@ +metadata: + name: require-approval-for-restarts +spec: + eventType: PreToolUse + hook: + type: prompt + prompt: If this action will restart or scale a resource, require human approval + before proceeding. + matcher: ^(restart_|scale_).* + permissionDecision: allow + enabled: true diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/repos/github-repo.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/repos/github-repo.yaml new file mode 100644 index 000000000..b29c262d3 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/repos/github-repo.yaml @@ -0,0 +1,5 @@ +name: github-repo +spec: + url: "{{githubRepo}}" + branch: main + description: Connected GitHub repository diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.md b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.md new file mode 100644 index 000000000..9503d3f3f --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.md @@ -0,0 +1,224 @@ +# Alert Noise Filter — Triage, Classify, and Act + +You are filtering Azure Monitor alert noise. Your goal: identify transient (noisy) alerts and close them without creating tickets, while escalating real persistent alerts to a full investigation and ServiceNow incident. + +--- + +## Phase 1: Extract Alert Context + +From the alert payload, extract: +- Alert rule name +- Alert rule ID +- Affected resource ID and resource type +- Severity (Sev0–Sev4) +- Fired timestamp (UTC) +- Monitor condition (should be "Fired") +- Subscription ID and resource group + +--- + +## Phase 1.5: Check for Correlated Alerts + +Before running the 15-minute check, determine if this alert is part of a broader incident by checking for other fired alerts AND verifying they share a dependency chain. + +### Step 1: Query all fired alerts in the same resource group + +``` +alertsmanagementresources +| where type == 'microsoft.alertsmanagement/alerts' +| where properties.essentials.monitorCondition == 'Fired' +| where properties.essentials.targetResourceGroup contains '' +| project name, alertRule=properties.essentials.alertRule, + resource=properties.essentials.targetResource, + resourceType=properties.essentials.targetResourceType, + severity=properties.essentials.severity, + firedTime=properties.essentials.startDateTime +| order by severity asc +``` + +If only one alert is fired, skip to Phase 2. + +### Step 2: Check dependency chain via App Insights + +If multiple alerts are fired, verify they're actually related by querying App Insights for dependency failures: + +``` +dependencies +| where timestamp > ago(30m) +| where success == false +| summarize failCount=count() by target, type, resultCode +| order by failCount desc +| take 10 +``` + +Also check the Application Map for the dependency chain: +``` +requests +| where timestamp > ago(30m) +| where success == false +| join kind=inner (dependencies | where timestamp > ago(30m) | where success == false) on operation_Id +| summarize count() by source=cloud_RoleName, target=dependency_target, depType=dependency_type +| order by count_ desc +``` + +### Step 3: Classify correlation + +**CORRELATED** — Group alerts together if: +- They are on resources in the same dependency chain (e.g., Container App → PostgreSQL, Gateway → Backend Service) +- The App Insights dependency failures show a common downstream failure (e.g., all services failing on the same DB connection) +- They fired within 30 minutes of each other + +**NOT CORRELATED** — Keep alerts separate if: +- The affected resources are not in the same dependency chain +- The error types are fundamentally different (e.g., CPU spike on one service vs certificate error on another) +- No shared dependency failures in App Insights + +### Step 4: Act on correlation + +For correlated alerts: +1. **Pick the highest-severity alert as the PRIMARY** — this is the one to investigate +2. **Pick the root-cause resource** — if a database is in the dependency chain and it's down, that's the root cause even if its alert is lower severity +3. **For all other correlated alerts**, close them immediately with: + ``` + CORRELATED ALERT — Closed + This alert is correlated with [PRIMARY_ALERT_NAME] (Sev[X]). + Dependency chain: [resource1] → [resource2] → [failed_resource] + Common failure: [dependency failure from App Insights] + See primary incident for investigation and ServiceNow ticket. + ``` +4. **Continue with Phase 2 ONLY for the primary alert** + +This prevents the agent from running multiple parallel 15-minute checks and investigations for the same underlying issue. + +--- + +## Phase 2: 15-Minute Transient Check + +This phase determines if the alert is transient noise or a real persistent issue. + +### Step 1: Check if alert is currently fired +Query the alert's monitor condition to confirm it's still in "Fired" state. Example: +``` +az monitor metrics alert show --name "" -g "" --query "isFiring" +``` +Or query fired alert instances for the subscription. + +If already resolved → classify as **noisy**, skip to Phase 3. + +### Step 2: Start the 15-minute observation window +Call the `wait-and-recheck-timer` tool with `wait_minutes: 15`. + +If unavailable, fall back to `ExecutePythonCode`: +```python +import time +for i in range(15): + time.sleep(60) +``` + +Do NOT poll or check alert state during the wait. Just wait for the tool to return. + +### Step 3: Re-check alert state +After the timer completes, check the alert's monitor condition again (same approach as Step 1). + +- Still fired → classify as **real** (proceed to Phase 4) +- Resolved → classify as **noisy** (proceed to Phase 3) + +--- + +## Phase 3: Classify — Noisy vs Real + +### NOISY (Transient) — Alert resolved within 15 minutes + +The alert auto-resolved. This is noise. Actions: +1. Close the incident immediately with this summary: + ``` + TRANSIENT ALERT — Closed (noise) + Alert: [name] | Resource: [resource] | Severity: [sev] + Fired: [timestamp] | Resolved: within 15-minute observation window + Classification: NOISY — no underlying issue detected + Action: None required. No ServiceNow ticket created. + ``` +2. **Do NOT** investigate further. +3. **Do NOT** create a ServiceNow ticket. +4. Done. + +### REAL (Persistent) — Alert still fired after 15 minutes + +The alert persisted. This is a real issue. Continue to Phase 4. + +--- + +## Phase 4: Investigate the Real Alert + +Now perform a full investigation: + +1. **Application Insights** — Query exceptions, failed requests, and dependency failures in the time window (30 min before alert to now): + - `exceptions | where timestamp > ago(45m) | summarize count() by type, outerMessage | top 10 by count_` + - `requests | where success == false | summarize count() by name, resultCode | top 10 by count_` + +2. **Log Analytics** — Query correlated logs: + - Container logs: `ContainerAppConsoleLogs_CL | where TimeGenerated > ago(45m) | where Log_s contains "error" or Log_s contains "exception"` + - App Service logs: `AppServiceHTTPLogs | where ScStatus >= 500` + +3. **Azure CLI** — Check for recent changes: + - `az monitor activity-log list -g --offset 24h --query "[?status.value=='Succeeded' && (operationName.value contains 'deploy' || operationName.value contains 'write')]"` + +4. **Metrics** — Use `PlotAreaChartWithCorrelation` to chart error rate, CPU, memory, or request count over the alert window. + +5. **Root Cause** — Based on evidence, determine: + - Category: Deployment / Configuration / Capacity / Code Bug / Infrastructure / External Dependency + - Blast radius: how many users/services affected + - Specific evidence (trace IDs, error messages, metric values) + +--- + +## Phase 5: Create ServiceNow Incident Ticket + +Use `servicenow-mcp_create_incident` with: + +| Field | Value | +|-------|-------| +| `short_description` | `[Sev{X}] {Alert Name} on {resource} — {root cause one-liner}` | +| `description` | Full report (see template below) | +| `urgency` | Sev0-1 → `1`, Sev2 → `2`, Sev3-4 → `3` | +| `impact` | Enterprise-wide → `1`, Department → `2`, Individual → `3` | + +**Description template:** +``` +PERSISTENT ALERT — Investigation Report +======================================== +Alert: {alert_name} +Resource: {resource} +Severity: {severity} +Fired: {timestamp} UTC +Transient Check: FAILED — still firing after 15-minute window + +TIMELINE +-------- +{chronological events from activity log and metrics} + +EVIDENCE +-------- +{App Insights exceptions, failed requests, Log Analytics errors} +{Metric chart descriptions} + +ROOT CAUSE +---------- +Category: {category} +{explanation backed by evidence} + +IMPACT +------ +Users affected: {estimate} +Services affected: {list} + +REMEDIATION +----------- +1. {specific action with command} +2. {verification step} +``` + +Then use `servicenow-mcp_add_work_note` with: +- `table`: "incident" +- `sys_id`: the sys_id from the created incident +- `note`: any additional evidence, chart data, or follow-up observations diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.yaml new file mode 100644 index 000000000..508e29451 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/skills/alert-noise-filter.yaml @@ -0,0 +1,15 @@ +metadata: + name: alert-noise-filter + description: "Triages Azure Monitor alerts: runs a 15-minute transient check to classify as noisy vs real, closes noisy alerts, and for real alerts performs investigation and creates a ServiceNow incident ticket." + spec: + tools: + - wait-and-recheck-timer + - RunAzCliReadCommands + - QueryAppInsightsUsingAppId + - QueryLogAnalyticsByWorkspaceId + - ExecutePythonCode + - PlotAreaChartWithCorrelation + - PlotBarChart + - SearchMemory +skillContent: skills/alert-noise-filter.md +additionalFiles: [] diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.instructions.md b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.instructions.md new file mode 100644 index 000000000..51bab252c --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.instructions.md @@ -0,0 +1,7 @@ +You are the Alert Noise Filter agent. When an Azure Monitor alert fires, follow the `alert-noise-filter` skill to triage it. + +## Key Behaviors +- Use the `wait-and-recheck-timer` tool for the observation window. If unavailable, use `ExecutePythonCode` with `time.sleep` as fallback. +- Do NOT poll or repeatedly check alert state while waiting. Call the timer once and wait for it to return. +- Do NOT create a ServiceNow ticket for transient/noisy alerts. +- ALWAYS investigate and create a ServiceNow ticket for persistent alerts. diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.yaml new file mode 100644 index 000000000..33848fda9 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/subagents/alert-triage-agent.yaml @@ -0,0 +1,20 @@ +metadata: + name: alert-triage-agent +spec: + instructions: subagents/alert-triage-agent.instructions.md + handoffDescription: Filters noisy Azure Monitor alerts using a 15-minute transient check. Closes transient alerts; investigates persistent ones and creates ServiceNow tickets. + handoffs: [] + tools: + - wait-and-recheck-timer + - ExecutePythonCode + - RunAzCliReadCommands + - QueryAppInsightsUsingAppId + - QueryLogAnalyticsByWorkspaceId + - SearchMemory + - PlotAreaChartWithCorrelation + - PlotBarChart + agentType: Autonomous + temperature: 0.2 + enableSkills: true + allowedSkills: + - alert-noise-filter diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/tools/wait-and-recheck-timer.yaml b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/tools/wait-and-recheck-timer.yaml new file mode 100644 index 000000000..c08c3dd07 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/config/tools/wait-and-recheck-timer.yaml @@ -0,0 +1,35 @@ +metadata: + name: wait-and-recheck-timer +spec: + type: PythonFunctionTool + name: wait-and-recheck-timer + description: "Waits the specified number of minutes. Use this to implement a deterministic observation window before re-checking alert state. Returns after the wait completes." + parameters: + - name: wait_minutes + type: integer + description: "Number of minutes to wait (default: 15)" + required: false + - name: reason + type: string + description: "Why we are waiting (logged for observability)" + required: false + timeout_seconds: 900 + dependencies: [] + function_code: | + import time + + def main(wait_minutes: int = 15, reason: str = "Observation window") -> dict: + print(f"[TIMER] Starting {wait_minutes}-minute wait. Reason: {reason}") + + for minute in range(wait_minutes): + time.sleep(60) + remaining = wait_minutes - 1 - minute + if remaining > 0: + print(f"[TIMER] {remaining} min remaining...") + + print(f"[TIMER] {wait_minutes}-minute wait complete.") + return { + "status": "complete", + "waited_minutes": wait_minutes, + "reason": reason + } diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/connectors.json b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/connectors.json new file mode 100644 index 000000000..d55bc43e3 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/connectors.json @@ -0,0 +1,36 @@ +{ + "toggles": { + "enableAppInsightsConnector": "{{appInsightsId:bool}}", + "appInsightsResourceId": "{{appInsightsId}}", + "appInsightsAppId": "{{appInsightsAppId}}", + "enableLogAnalyticsConnector": "{{lawId:bool}}", + "lawResourceId": "{{lawId}}", + "enableAzureMonitorConnector": false, + "azureMonitorLookbackDays": 7, + "grafanaUrl": "", + "grafanaApiKey": "" + }, + "connectors": [ + { + "name": "servicenow-mcp", + "properties": { + "dataConnectorType": "Mcp", + "dataSource": "placeholder", + "extendedProperties": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@onlyflows/servicenow-mcp"], + "envs": { + "SN_INSTANCE": "https://{{snowInstance}}.service-now.com", + "SN_USER": "{{snowUser}}", + "SN_PASSWORD": "${SERVICENOW_PASSWORD}" + }, + "toolsVisibleToMetaAgent": ["*"], + "selectedTools": ["create_incident", "update_incident", "add_work_note", "get_incident", "query_records", "search_knowledge", "resolve_incident"] + }, + "identity": "system" + } + } + ] +} + diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/data/incident-report-template.md b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/data/incident-report-template.md new file mode 100644 index 000000000..1c05b9733 --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/data/incident-report-template.md @@ -0,0 +1,65 @@ +# Incident Report Template — Transient Alert Check Flow + +## Summary + + +**Alert Rule**: [Azure Monitor alert rule name] +**Severity**: [Sev0-4] +**Resource**: [Affected Azure resource ID] +**Fired At**: [UTC timestamp] +**Transient Check Result**: [TRANSIENT — self-resolved / PERSISTENT — still firing after 15 min] +**Status**: [Closed-Transient / Investigating / Resolved / Mitigated] +**ServiceNow Ticket**: [INC number, if created] + +--- + +## Transient Check Details + +| Step | Time (UTC) | Result | +|---|---|---| +| Alert fired | HH:MM | Monitor condition: Fired | +| 15-min timer started | HH:MM | Observation window opened | +| 15-min timer completed | HH:MM | Re-checked alert state | +| Alert state at check | HH:MM | [Fired / Resolved] | +| Decision | HH:MM | [Closed as transient / Escalated to investigation] | + +--- + +## Investigation (only for persistent alerts) + +### Evidence + +#### Azure Monitor Metrics + + +#### Application Insights + + +#### Log Analytics + + +#### Activity Log + + +--- + +### Root Cause + + + +**Category**: [Database / Network / Deployment / Configuration / Code Bug / Capacity] + +--- + +### Remediation + +#### Recommended Actions +1. [Specific action with exact command] +2. [Verification step] + +#### ServiceNow Ticket Details +- **Ticket ID**: [INC number] +- **Urgency**: [1-High / 2-Medium / 3-Low] +- **Impact**: [1-High / 2-Medium / 3-Low] +- **Assignment Group**: [Team name] +- **Description**: [Summary of analysis included in ticket] diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/expected-config.json b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/expected-config.json new file mode 100644 index 000000000..447d4eb3b --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/expected-config.json @@ -0,0 +1,49 @@ +{ + "_description": "Expected configuration for azmon-alert-noise-filter-snow. Used by verify-agent.sh to validate deployments.", + "_scenario": "azmon-alert-noise-filter-snow", + + "agent": { + "accessLevel": "High", + "actionMode": "Review", + "upgradeChannel": "Preview", + "defaultModelProvider": "Anthropic", + "incidentPlatform": "AzMonitor" + }, + + "connectors": [ + { "name": "app-insights", "type": "AppInsights" }, + { "name": "log-analytics", "type": "LogAnalytics" }, + { "name": "servicenow-mcp", "type": "Mcp" } + ], + + "skills": [ + "alert-noise-filter" + ], + + "subagents": [ + "alert-triage-agent" + ], + + "tools": [ + { "name": "wait-and-recheck-timer", "type": "PythonFunctionTool" } + ], + + "hooks": [ + "deny-prod-deletes", + "require-approval-for-restarts", + "verify-alert-state-before-action" + ], + + "commonPrompts": [ + "investigation-guidelines", + "safety-rules" + ], + + "scheduledTasks": [], + + "responsePlans": [ + { "name": "azmon-all-severities", "handlingAgent": "alert-triage-agent", "mergeEnabled": true, "mergeWindowHours": 1 } + ], + + "repos": [] +} diff --git a/sreagent-templates/recipes/azmon-alert-noise-filter-snow/tool-permissions.json b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/tool-permissions.json new file mode 100644 index 000000000..e9cd3114c --- /dev/null +++ b/sreagent-templates/recipes/azmon-alert-noise-filter-snow/tool-permissions.json @@ -0,0 +1,13 @@ +{ + "allow": [ + "RunInTerminal", + "ExecutePythonCode" + ], + "ask": [ + "RunAzCliWriteCommands" + ], + "deny": [ + "FetchWebpage", + "SearchWebpage" + ] +}