diff --git a/probe-cli/README.md b/probe-cli/README.md new file mode 100644 index 0000000..730ca64 --- /dev/null +++ b/probe-cli/README.md @@ -0,0 +1,155 @@ +# probe-cli + +Automates the [Site Archaeology protocol](https://github.com/better-world-ai/agent-cli-creator) from the x-cli ecosystem. + +One command replaces 30-60 minutes of manual `curl` calls with 30 seconds of auto-detection. + +## What it does + +Given any URL, probe-cli connects to the kimi-webbridge daemon and automatically: + +1. **Detects auth mechanism** — scans localStorage, sessionStorage, and cookies for token/session/csrf patterns +2. **Extracts DOM elements** — finds all forms, inputs, buttons with their selectors +3. **Captures network traffic** — filters XHR/Fetch requests, inspects auth headers, identifies API endpoints +4. **Recommends a CLI pattern** — scores dom-scrape / api-reverse / form-submit / async-poll based on findings +5. **Outputs a JSON site profile** — ready for the agent-cli-creator Phase 4 (implementation) + +## Protocol Mapping + +| Site Archaeology Step | probe-cli automation | +|-----------------------|---------------------| +| Step 1: Navigate | ✅ Auto-navigate to URL | +| Step 2: Snapshot DOM | ✅ Auto-extract forms, inputs, buttons | +| Step 3: Start network capture | ✅ Auto-start before navigate | +| Step 4: Stop + list requests | ✅ Auto-stop after page settles | +| Step 5: Inspect request detail | ✅ Auto-inspect auth headers | +| Step 6: Verify with evaluate | ✅ Auth pattern detection + recommendation | + +## Install + +```bash +# Build from source +git clone https://github.com/RachelXiaolan/probe-cli.git +cd probe-cli +go build -o probe-cli . +``` + +## Usage + +```bash +# Full probe (auth + DOM + network) +probe-cli https://example.com + +# Quick mode (auth + DOM only, skip network capture) +probe-cli https://example.com --quick + +# Custom session name +probe-cli https://example.com --session mysite +``` + +**Prerequisites:** kimi-webbridge daemon running at `http://127.0.0.1:10086` + +## Output Format + +```json +{ + "ok": true, + "data": { + "url": "https://fish.audio/zh-CN/app/text-to-speech/", + "timestamp": "2026-06-01T12:00:00+09:00", + "auth": { + "detected": true, + "method": "bearer-localstorage", + "storage_type": "localStorage", + "token_keys": ["token"] + }, + "dom": { + "forms": [], + "standalone_inputs": [ + {"tag": "div", "content_editable": true, "role": "textbox"} + ], + "buttons": [ + {"text": "生成语音", "type": "button"} + ] + }, + "network": { + "total_requests": 42, + "api_endpoints": [ + { + "method": "POST", + "url": "https://api.fish.audio/task", + "has_auth_header": true, + "auth_headers": ["Authorization: Bearer eyJhbG***"] + } + ], + "static_filtered": 37 + }, + "pattern": { + "type": "api-reverse", + "confidence": 0.85, + "reason": "Data loaded via XHR/Fetch API calls with bearer-localstorage auth. Replicate API calls in evaluate()." + } + } +} +``` + +## How it works + +``` +┌─────────────────────────────────────────┐ +│ 1. Start network capture │ +│ 2. Navigate to URL │ +│ 3. Wait 3s for page to settle │ +├─────────────────────────────────────────┤ +│ 4. Inject auth detection JS │ +│ → localStorage / cookie / sessionStorage scan │ +│ 5. Inject DOM extraction JS │ +│ → forms, inputs, buttons │ +├─────────────────────────────────────────┤ +│ 6. Stop network capture │ +│ 7. List + filter XHR/Fetch requests │ +│ 8. Inspect auth headers per endpoint │ +├─────────────────────────────────────────┤ +│ 9. Score patterns + recommend │ +│ 10. Output site profile JSON │ +└─────────────────────────────────────────┘ +``` + +## Integration with agent-cli-creator + +After probe-cli outputs a site profile, the AI agent can skip directly to Phase 4 (Implementation) of the agent-cli-creator workflow: + +``` +Before: Agent reads site-exploration.md → manually runs 6 curl steps → 30-60 min +After: Agent runs probe-cli → gets site-profile.json → skip to Phase 4 +``` + +The site profile contains everything the agent needs: +- `auth.method` → tells the agent how to authenticate +- `network.api_endpoints` → tells the agent which APIs to call +- `dom.forms/inputs/buttons` → tells the agent which elements to interact with +- `pattern.type` → tells the agent which template to use + +## Auth Detection Patterns + +| Detected Method | What it means | How CLI should handle | +|----------------|---------------|----------------------| +| `bearer-localstorage` | JWT/Bearer token in localStorage | Extract token via evaluate, add to fetch headers | +| `csrf-cookie` | CSRF token in cookie (e.g. ct0) | Extract from cookie, add x-csrf-token header | +| `cookie-only` | Session cookie, no extra headers | Just use evaluate (cookies sent automatically) | +| `session-storage` | Auth token in sessionStorage | Extract via evaluate, add to headers | +| `localstorage-other` | Non-token auth data in localStorage | Inspect keys for API key or session ID | +| *(none detected)* | No login required, or complex auth | Manual archaeology needed | + +## CLI Pattern Types + +| Pattern | When | Implementation approach | +|---------|------|------------------------| +| `dom-scrape` | Data visible in DOM, no API | Snapshot + evaluate to extract text | +| `api-reverse` | Data from XHR/Fetch with auth | Replicate fetch calls in evaluate | +| `form-submit` | Forms to fill and submit | Fill fields + click submit via evaluate | +| `async-poll` | POST action → poll GET for result | Submit + loop poll until done | + +## License + +MIT diff --git a/probe-cli/browser/client.go b/probe-cli/browser/client.go new file mode 100644 index 0000000..ded0db8 --- /dev/null +++ b/probe-cli/browser/client.go @@ -0,0 +1,102 @@ +package browser + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" +) + +const DefaultDaemonURL = "http://127.0.0.1:10086" + +type Client struct { + baseURL string + session string + http *http.Client +} + +func NewClient(session string) *Client { + return &Client{ + baseURL: DefaultDaemonURL, + session: session, + http: &http.Client{Timeout: 90 * time.Second}, + } +} + +func (c *Client) Call(action string, args map[string]any) (json.RawMessage, error) { + body, _ := json.Marshal(map[string]any{ + "action": action, + "session": c.session, + "args": args, + }) + resp, err := c.http.Post(c.baseURL+"/command", "application/json", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("daemon unreachable: %w", err) + } + defer resp.Body.Close() + + var result struct { + OK bool `json:"ok"` + Data json.RawMessage `json:"data"` + Error *struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + if !result.OK { + if result.Error != nil { + return nil, fmt.Errorf("%s: %s", result.Error.Code, result.Error.Message) + } + return nil, fmt.Errorf("unknown error from daemon") + } + return result.Data, nil +} + +// Quick helpers + +func (c *Client) Navigate(url string) error { + _, err := c.Call("navigate", map[string]any{"url": url, "newTab": true}) + return err +} + +func (c *Client) Evaluate(code string) (json.RawMessage, error) { + return c.Call("evaluate", map[string]any{"code": code}) +} + +func (c *Client) Snapshot() (json.RawMessage, error) { + return c.Call("snapshot", map[string]any{}) +} + +func (c *Client) NetworkStart() error { + _, err := c.Call("network", map[string]any{"cmd": "start"}) + return err +} + +func (c *Client) NetworkStop() error { + _, err := c.Call("network", map[string]any{"cmd": "stop"}) + return err +} + +func (c *Client) NetworkList() (json.RawMessage, error) { + return c.Call("network", map[string]any{"cmd": "list"}) +} + +func (c *Client) NetworkDetail(requestID string) (json.RawMessage, error) { + return c.Call("network", map[string]any{"cmd": "detail", "requestId": requestID}) +} + +// Status checks if the kimi-webbridge daemon is running. +func (c *Client) Status() (json.RawMessage, error) { + resp, err := c.http.Get(c.baseURL + "/status") + if err != nil { + return nil, fmt.Errorf("daemon unreachable at %s: %w", c.baseURL, err) + } + defer resp.Body.Close() + var result json.RawMessage + json.NewDecoder(resp.Body).Decode(&result) + return result, nil +} diff --git a/probe-cli/cmd/root.go b/probe-cli/cmd/root.go new file mode 100644 index 0000000..854ecc4 --- /dev/null +++ b/probe-cli/cmd/root.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "probe-cli/browser" + "probe-cli/output" + "probe-cli/probe" +) + +var ( + sessionName string + quickMode bool +) + +var rootCmd = &cobra.Command{ + Use: "probe-cli ", + Short: "Auto-detect site auth, API endpoints, and DOM elements for x-cli development", + Long: `probe-cli automates the Site Archaeology protocol from agent-cli-creator. + +It connects to the kimi-webbridge daemon, navigates to the target URL, +and auto-detects: + - Authentication mechanism (localStorage, cookie, sessionStorage, CSRF) + - API endpoints (XHR/Fetch with auth header analysis) + - DOM elements (forms, inputs, buttons) + - Recommended CLI pattern (dom-scrape, api-reverse, form-submit, async-poll) + +Output is a JSON site profile ready for CLI implementation.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + targetURL := args[0] + + // Use URL hostname as session name if not specified + sess := sessionName + if sess == "" { + sess = "probe" + } + + client := browser.NewClient(sess) + + // Check daemon is running + if _, err := client.Status(); err != nil { + output.Error("daemon_unreachable", fmt.Sprintf( + "kimi-webbridge daemon not running at %s. Start it first.", + browser.DefaultDaemonURL, + )) + os.Exit(1) + } + + var profile *probe.SiteProfile + var err error + + if quickMode { + profile, err = probe.RunQuick(client, targetURL) + } else { + profile, err = probe.Run(client, targetURL) + } + + if err != nil { + output.Error("probe_failed", err.Error()) + os.Exit(1) + } + + output.Success(profile) + }, +} + +func Execute() error { + return rootCmd.Execute() +} + +func init() { + rootCmd.Flags().StringVarP(&sessionName, "session", "s", "", "kimi-webbridge session name (default: probe)") + rootCmd.Flags().BoolVarP(&quickMode, "quick", "q", false, "skip network capture (DOM + auth only)") +} diff --git a/probe-cli/go.mod b/probe-cli/go.mod new file mode 100644 index 0000000..e9c7797 --- /dev/null +++ b/probe-cli/go.mod @@ -0,0 +1,9 @@ +module probe-cli + +go 1.21 + +require github.com/spf13/cobra v1.8.0 + +require github.com/inconshreveable/mousetrap v1.1.0 // indirect + +require github.com/spf13/pflag v1.0.5 // indirect diff --git a/probe-cli/go.sum b/probe-cli/go.sum new file mode 100644 index 0000000..d0e8c2c --- /dev/null +++ b/probe-cli/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/probe-cli/main.go b/probe-cli/main.go new file mode 100644 index 0000000..e2b2096 --- /dev/null +++ b/probe-cli/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "os" + + "probe-cli/cmd" +) + +func main() { + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/probe-cli/output/output.go b/probe-cli/output/output.go new file mode 100644 index 0000000..4341740 --- /dev/null +++ b/probe-cli/output/output.go @@ -0,0 +1,24 @@ +package output + +import ( + "encoding/json" + "fmt" + "os" +) + +func Success(data any) { + printJSON(map[string]any{"ok": true, "data": data}) +} + +func Error(code, message string) { + printJSON(map[string]any{"ok": false, "error": map[string]any{"code": code, "message": message}}) +} + +func printJSON(v any) { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(v); err != nil { + fmt.Fprintf(os.Stderr, "output error: %v\n", err) + } +} diff --git a/probe-cli/probe/auth.go b/probe-cli/probe/auth.go new file mode 100644 index 0000000..87a80c7 --- /dev/null +++ b/probe-cli/probe/auth.go @@ -0,0 +1,160 @@ +package probe + +import ( + "encoding/json" + "fmt" + "strings" + + "probe-cli/browser" +) + +// authDetectJS scans localStorage, sessionStorage, and cookies for auth-related keys. +const authDetectJS = `(() => { + const AUTH_PATTERNS = [ + 'token', 'access_token', 'auth_token', 'jwt', 'bearer', + 'session', 'user_id', 'uid', 'api_key', 'csrf', 'ct0', + 'xt', 'sid', 'refresh', 'id_token', 'identity' + ]; + + const result = { + localStorage: {}, + sessionStorage: {}, + cookies: {}, + globals: {}, + detected_type: null + }; + + // Helper: check if key matches any auth pattern + function isAuthKey(key) { + const lower = key.toLowerCase(); + return AUTH_PATTERNS.some(p => lower.includes(p)); + } + + // Scan localStorage + try { + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i); + if (isAuthKey(key)) { + result.localStorage[key] = localStorage.getItem(key).substring(0, 80); + } + } + } catch(e) {} + + // Scan sessionStorage + try { + for (let i = 0; i < sessionStorage.length; i++) { + const key = sessionStorage.key(i); + if (isAuthKey(key)) { + result.sessionStorage[key] = sessionStorage.getItem(key).substring(0, 80); + } + } + } catch(e) {} + + // Scan cookies + try { + document.cookie.split(';').forEach(c => { + const eq = c.indexOf('='); + if (eq < 0) return; + const name = c.substring(0, eq).trim(); + if (isAuthKey(name)) { + result.cookies[name] = c.substring(eq + 1).trim().substring(0, 80); + } + }); + } catch(e) {} + + // Check framework globals + if (window.__NEXT_DATA__) result.globals['__NEXT_DATA__'] = true; + if (window.__NUXT__) result.globals['__NUXT__'] = true; + + // Detect auth type + const lsKeys = Object.keys(result.localStorage); + const ssKeys = Object.keys(result.sessionStorage); + const ckKeys = Object.keys(result.cookies); + + if (lsKeys.some(k => k.toLowerCase().includes('token'))) { + result.detected_type = 'bearer-localstorage'; + } else if (ckKeys.some(k => k.toLowerCase().includes('csrf') || k.toLowerCase().includes('ct0'))) { + result.detected_type = 'csrf-cookie'; + } else if (ckKeys.length > 0 && lsKeys.length === 0) { + result.detected_type = 'cookie-only'; + } else if (lsKeys.length > 0) { + result.detected_type = 'localstorage-other'; + } else if (ssKeys.length > 0) { + result.detected_type = 'session-storage'; + } + + return result; +})()` + +// rawAuthResult mirrors the JS return value. +type rawAuthResult struct { + LocalStorage map[string]string `json:"localStorage"` + SessionStorage map[string]string `json:"sessionStorage"` + Cookies map[string]string `json:"cookies"` + Globals map[string]bool `json:"globals"` + DetectedType *string `json:"detected_type"` +} + +// DetectAuth probes the page for authentication mechanisms. +func DetectAuth(client *browser.Client) (*AuthResult, []string) { + var warnings []string + + raw, err := client.Evaluate(authDetectJS) + if err != nil { + warnings = append(warnings, fmt.Sprintf("auth detection failed: %v", err)) + return &AuthResult{Detected: false}, warnings + } + + var parsed rawAuthResult + if err := json.Unmarshal(raw, &parsed); err != nil { + warnings = append(warnings, fmt.Sprintf("auth result parse failed: %v", err)) + return &AuthResult{Detected: false}, warnings + } + + result := &AuthResult{} + + // Collect all auth keys found + var allKeys []string + for k := range parsed.LocalStorage { + allKeys = append(allKeys, "localStorage:"+k) + } + for k := range parsed.SessionStorage { + allKeys = append(allKeys, "sessionStorage:"+k) + } + for k := range parsed.Cookies { + allKeys = append(allKeys, "cookie:"+k) + } + if len(allKeys) > 0 { + allJSON, _ := json.Marshal(allKeys) + result.AllKeys = allJSON + } + + // Determine method + if parsed.DetectedType != nil && *parsed.DetectedType != "" { + result.Detected = true + result.Method = *parsed.DetectedType + + // Extract token keys + switch { + case strings.Contains(result.Method, "localstorage"): + result.StorageType = "localStorage" + for k := range parsed.LocalStorage { + if strings.Contains(strings.ToLower(k), "token") { + result.TokenKeys = append(result.TokenKeys, k) + } + } + case strings.Contains(result.Method, "session"): + result.StorageType = "sessionStorage" + for k := range parsed.SessionStorage { + result.TokenKeys = append(result.TokenKeys, k) + } + case strings.Contains(result.Method, "cookie"): + result.StorageType = "cookie" + for k := range parsed.Cookies { + result.TokenKeys = append(result.TokenKeys, k) + } + } + } + + return result, warnings +} diff --git a/probe-cli/probe/dom.go b/probe-cli/probe/dom.go new file mode 100644 index 0000000..d2ca6f1 --- /dev/null +++ b/probe-cli/probe/dom.go @@ -0,0 +1,166 @@ +package probe + +import ( + "encoding/json" + "fmt" + + "probe-cli/browser" +) + +// domExtractJS scans the page for forms, inputs, and buttons. +const domExtractJS = `(() => { + const result = { forms: [], standalone_inputs: [], buttons: [] }; + + // Forms + document.querySelectorAll('form').forEach((f, i) => { + const inputs = Array.from(f.querySelectorAll('input, textarea, select')).map(el => ({ + tag: el.tagName.toLowerCase(), + type: el.getAttribute('type') || null, + name: el.getAttribute('name') || null, + placeholder: el.getAttribute('placeholder') || null, + id: el.id || null, + content_editable: el.contentEditable === 'true', + role: el.getAttribute('role') || null + })); + result.forms.push({ + index: i, + action: f.action || null, + method: (f.method || 'GET').toUpperCase(), + id: f.id || null, + input_count: inputs.length, + inputs: inputs + }); + }); + + // Standalone inputs (not inside a form) + document.querySelectorAll('input, textarea, select, [contenteditable="true"]').forEach(el => { + if (!el.closest('form')) { + result.standalone_inputs.push({ + tag: el.tagName.toLowerCase(), + type: el.getAttribute('type') || 'text', + name: el.getAttribute('name') || null, + placeholder: el.getAttribute('placeholder') || null, + id: el.id || null, + content_editable: el.contentEditable === 'true', + role: el.getAttribute('role') || null + }); + } + }); + + // Buttons (limit to 30 to avoid huge output) + const buttons = document.querySelectorAll('button, [role="button"], input[type="submit"]'); + const limit = Math.min(buttons.length, 30); + for (let i = 0; i < limit; i++) { + const el = buttons[i]; + result.buttons.push({ + text: (el.textContent || '').trim().substring(0, 80) || null, + type: el.getAttribute('type') || null, + id: el.id || null, + aria_label: el.getAttribute('aria-label') || null, + disabled: !!el.disabled + }); + } + + return result; +})()` + +type rawDOMResult struct { + Forms []rawFormInfo `json:"forms"` + StandaloneInputs []rawInputInfo `json:"standalone_inputs"` + Buttons []rawButtonInfo `json:"buttons"` +} + +type rawFormInfo struct { + Index int `json:"index"` + Action string `json:"action"` + Method string `json:"method"` + ID string `json:"id"` + InputCount int `json:"input_count"` + Inputs []rawInputInfo `json:"inputs"` +} + +type rawInputInfo struct { + Tag string `json:"tag"` + Type string `json:"type"` + Name string `json:"name"` + Placeholder string `json:"placeholder"` + ID string `json:"id"` + ContentEditable bool `json:"content_editable"` + Role string `json:"role"` +} + +type rawButtonInfo struct { + Text string `json:"text"` + Type string `json:"type"` + ID string `json:"id"` + AriaLabel string `json:"aria_label"` + Disabled bool `json:"disabled"` +} + +// ExtractDOM probes the page for interactive elements. +func ExtractDOM(client *browser.Client) (*DOMResult, []string) { + var warnings []string + + raw, err := client.Evaluate(domExtractJS) + if err != nil { + warnings = append(warnings, fmt.Sprintf("DOM extraction failed: %v", err)) + return &DOMResult{}, warnings + } + + var parsed rawDOMResult + if err := json.Unmarshal(raw, &parsed); err != nil { + warnings = append(warnings, fmt.Sprintf("DOM result parse failed: %v", err)) + return &DOMResult{}, warnings + } + + result := &DOMResult{} + + // Convert forms + for _, f := range parsed.Forms { + form := FormInfo{ + Index: f.Index, + Action: f.Action, + Method: f.Method, + ID: f.ID, + InputCount: f.InputCount, + } + for _, inp := range f.Inputs { + form.Inputs = append(form.Inputs, InputInfo{ + Tag: inp.Tag, + Type: inp.Type, + Name: inp.Name, + Placeholder: inp.Placeholder, + ID: inp.ID, + ContentEditable: inp.ContentEditable, + Role: inp.Role, + }) + } + result.Forms = append(result.Forms, form) + } + + // Convert standalone inputs + for _, inp := range parsed.StandaloneInputs { + result.StandaloneInputs = append(result.StandaloneInputs, InputInfo{ + Tag: inp.Tag, + Type: inp.Type, + Name: inp.Name, + Placeholder: inp.Placeholder, + ID: inp.ID, + ContentEditable: inp.ContentEditable, + Role: inp.Role, + }) + } + + // Convert buttons + for _, b := range parsed.Buttons { + result.Buttons = append(result.Buttons, ButtonInfo{ + Text: b.Text, + Type: b.Type, + ID: b.ID, + AriaLabel: b.AriaLabel, + Disabled: b.Disabled, + }) + } + + return result, warnings +} diff --git a/probe-cli/probe/network.go b/probe-cli/probe/network.go new file mode 100644 index 0000000..114cd1f --- /dev/null +++ b/probe-cli/probe/network.go @@ -0,0 +1,163 @@ +package probe + +import ( + "encoding/json" + "fmt" + "strings" + + "probe-cli/browser" +) + +// Static file extensions to filter out. +var staticExts = []string{ + ".js", ".css", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", + ".woff", ".woff2", ".ttf", ".eot", ".otf", + ".mp4", ".mp3", ".webm", ".webp", ".avif", +} + +// rawNetworkList matches the expected daemon response for "list" command. +type rawNetworkItem struct { + RequestID string `json:"requestId"` + URL string `json:"url"` + Method string `json:"method"` + Status int `json:"status"` + Type string `json:"type"` // XHR, Fetch, Document, Script, Stylesheet, Image, etc. +} + +// rawNetworkDetail matches the expected daemon response for "detail" command. +type rawNetworkDetail struct { + RequestID string `json:"requestId"` + URL string `json:"url"` + Method string `json:"method"` + Status int `json:"status"` + RequestHeaders map[string]string `json:"requestHeaders"` + RequestBody string `json:"requestBody"` + ResponseHeaders map[string]string `json:"responseHeaders"` + ContentType string `json:"contentType"` +} + +// AnalyzeNetwork stops capture, lists requests, filters to API calls, and inspects details. +func AnalyzeNetwork(client *browser.Client) (*NetworkResult, []string) { + var warnings []string + + // Get list of captured requests + rawList, err := client.NetworkList() + if err != nil { + warnings = append(warnings, fmt.Sprintf("network list failed: %v", err)) + return &NetworkResult{}, warnings + } + + var items []rawNetworkItem + if err := json.Unmarshal(rawList, &items); err != nil { + // Try as map with "requests" key + var wrapper struct { + Requests []rawNetworkItem `json:"requests"` + } + if err2 := json.Unmarshal(rawList, &wrapper); err2 != nil { + warnings = append(warnings, fmt.Sprintf("network list parse failed: %v", err)) + return &NetworkResult{}, warnings + } + items = wrapper.Requests + } + + result := &NetworkResult{ + TotalRequests: len(items), + } + + // Filter: keep only XHR and Fetch requests + var apiItems []rawNetworkItem + for _, item := range items { + if isStaticURL(item.URL) { + result.StaticFiltered++ + continue + } + typ := strings.ToUpper(item.Type) + if typ == "XHR" || typ == "FETCH" || typ == "" { + // Empty type might be from older daemon versions; include and check later + apiItems = append(apiItems, item) + } + } + + // Deduplicate by URL (keep first occurrence per unique URL) + seen := make(map[string]bool) + var uniqueAPI []rawNetworkItem + for _, item := range apiItems { + if !seen[item.URL] { + seen[item.URL] = true + uniqueAPI = append(uniqueAPI, item) + } + } + + // Get details for up to 15 unique API endpoints + limit := 15 + if len(uniqueAPI) < limit { + limit = len(uniqueAPI) + } + + for i := 0; i < limit; i++ { + item := uniqueAPI[i] + endpoint := APIEndpoint{ + Method: item.Method, + URL: item.URL, + Status: item.Status, + } + + // Try to get details for auth header inspection + detail, err := client.NetworkDetail(item.RequestID) + if err != nil { + // Detail failed — still include the endpoint with basic info + result.APIEndpoints = append(result.APIEndpoints, endpoint) + continue + } + + var d rawNetworkDetail + if err := json.Unmarshal(detail, &d); err != nil { + result.APIEndpoints = append(result.APIEndpoints, endpoint) + continue + } + + // Check for auth-related headers + for key, val := range d.RequestHeaders { + lower := strings.ToLower(key) + if strings.Contains(lower, "auth") || strings.Contains(lower, "token") || + strings.Contains(lower, "csrf") || strings.Contains(lower, "x-csrf") { + endpoint.HasAuthHeader = true + // Mask token value for safety (show first 8 chars + ...) + masked := maskSecret(val) + endpoint.AuthHeaders = append(endpoint.AuthHeaders, fmt.Sprintf("%s: %s", key, masked)) + } + } + + if ct, ok := d.RequestHeaders["Content-Type"]; ok { + endpoint.ContentType = ct + } + + result.APIEndpoints = append(result.APIEndpoints, endpoint) + } + + return result, warnings +} + +// isStaticURL returns true for static asset URLs. +func isStaticURL(url string) bool { + // Strip query string for extension check + path := url + if idx := strings.Index(url, "?"); idx >= 0 { + path = url[:idx] + } + lower := strings.ToLower(path) + for _, ext := range staticExts { + if strings.HasSuffix(lower, ext) { + return true + } + } + return false +} + +// maskSecret hides most of a secret value, keeping just a prefix. +func maskSecret(val string) string { + if len(val) <= 12 { + return "***" + } + return val[:8] + "***" +} diff --git a/probe-cli/probe/pattern.go b/probe-cli/probe/pattern.go new file mode 100644 index 0000000..ce95477 --- /dev/null +++ b/probe-cli/probe/pattern.go @@ -0,0 +1,135 @@ +package probe + +import ( + "strings" +) + +// DetectPattern analyzes auth, DOM, and network results to recommend a CLI pattern. +func DetectPattern(auth *AuthResult, dom *DOMResult, network *NetworkResult) *PatternResult { + score := map[string]float64{ + "dom-scrape": 0, + "api-reverse": 0, + "form-submit": 0, + "async-poll": 0, + } + + // ── Signal: API endpoints found ── + if network != nil && len(network.APIEndpoints) > 0 { + score["api-reverse"] += 0.4 + + // If multiple endpoints with auth headers → strong API-reverse signal + authCount := 0 + for _, ep := range network.APIEndpoints { + if ep.HasAuthHeader { + authCount++ + } + } + if authCount > 0 { + score["api-reverse"] += 0.3 + } + + // If there's a POST followed by GET with same base → async-poll pattern + hasPost := false + hasGet := false + for _, ep := range network.APIEndpoints { + if ep.Method == "POST" { + hasPost = true + } + if ep.Method == "GET" { + hasGet = true + } + } + if hasPost && hasGet { + score["async-poll"] += 0.2 + } + } + + // ── Signal: Auth detected ── + if auth != nil && auth.Detected { + switch { + case strings.Contains(auth.Method, "bearer") || strings.Contains(auth.Method, "localstorage"): + score["api-reverse"] += 0.2 + case strings.Contains(auth.Method, "csrf"): + score["api-reverse"] += 0.15 + case strings.Contains(auth.Method, "cookie"): + score["api-reverse"] += 0.1 + } + } + + // ── Signal: DOM has forms ── + if dom != nil { + if len(dom.Forms) > 0 { + score["form-submit"] += 0.3 + // Forms with few inputs (search box) → simpler pattern + for _, f := range dom.Forms { + if f.InputCount <= 2 && f.Method == "GET" { + score["dom-scrape"] += 0.1 + } + } + } + // Standalone contenteditable → rich text / input for generation + for _, inp := range dom.StandaloneInputs { + if inp.ContentEditable { + score["form-submit"] += 0.15 + score["async-poll"] += 0.1 + } + } + // Buttons with submit/generate keywords → async-poll hint + for _, btn := range dom.Buttons { + lower := strings.ToLower(btn.Text) + if strings.Contains(lower, "生成") || strings.Contains(lower, "generate") || + strings.Contains(lower, "create") || strings.Contains(lower, "submit") { + score["async-poll"] += 0.15 + score["form-submit"] += 0.1 + } + } + } + + // ── Signal: No API endpoints found → DOM scrape is the only option ── + if network == nil || len(network.APIEndpoints) == 0 { + score["dom-scrape"] += 0.5 + } + + // ── Pick highest scoring pattern ── + best := "dom-scrape" + bestScore := 0.0 + for pattern, s := range score { + if s > bestScore { + bestScore = s + best = pattern + } + } + + // Cap confidence at 0.95 + if bestScore > 0.95 { + bestScore = 0.95 + } + + reason := patternReason(best, auth, dom, network) + + return &PatternResult{ + Type: best, + Confidence: bestScore, + Reason: reason, + } +} + +func patternReason(pattern string, auth *AuthResult, dom *DOMResult, network *NetworkResult) string { + switch pattern { + case "dom-scrape": + return "Data is available in page DOM; no API calls detected. Use snapshot + evaluate to extract." + case "api-reverse": + reason := "Data loaded via XHR/Fetch API calls" + if auth != nil && auth.Detected { + reason += " with " + auth.Method + " auth" + } + reason += ". Replicate API calls in evaluate()." + return reason + case "form-submit": + return "Page has forms that accept user input. Fill form fields + submit via evaluate()." + case "async-poll": + return "Action triggers async processing (POST → poll GET for result). Implement submit + poll loop." + default: + return "Pattern unclear. Manual site archaeology recommended." + } +} diff --git a/probe-cli/probe/probe.go b/probe-cli/probe/probe.go new file mode 100644 index 0000000..71ec017 --- /dev/null +++ b/probe-cli/probe/probe.go @@ -0,0 +1,159 @@ +package probe + +import ( + "encoding/json" + "fmt" + "time" + + "probe-cli/browser" +) + +// SiteProfile is the final output of a probe run. +type SiteProfile struct { + URL string `json:"url"` + Timestamp string `json:"timestamp"` + Auth *AuthResult `json:"auth"` + DOM *DOMResult `json:"dom"` + Network *NetworkResult `json:"network"` + Pattern *PatternResult `json:"pattern"` + Warnings []string `json:"warnings,omitempty"` +} + +// AuthResult holds auth detection findings. +type AuthResult struct { + Detected bool `json:"detected"` + Method string `json:"method,omitempty"` // bearer-localstorage, csrf-cookie, cookie-only, session-storage, none + StorageType string `json:"storage_type,omitempty"` // localStorage, sessionStorage, cookie + TokenKeys []string `json:"token_keys,omitempty"` + AllKeys json.RawMessage `json:"all_keys,omitempty"` + LoginURL string `json:"login_url,omitempty"` +} + +// DOMResult holds extracted page element info. +type DOMResult struct { + Forms []FormInfo `json:"forms"` + StandaloneInputs []InputInfo `json:"standalone_inputs"` + Buttons []ButtonInfo `json:"buttons"` +} + +type FormInfo struct { + Index int `json:"index"` + Action string `json:"action,omitempty"` + Method string `json:"method"` + ID string `json:"id,omitempty"` + InputCount int `json:"input_count"` + Inputs []InputInfo `json:"inputs"` +} + +type InputInfo struct { + Tag string `json:"tag"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Placeholder string `json:"placeholder,omitempty"` + ID string `json:"id,omitempty"` + ContentEditable bool `json:"content_editable"` + Role string `json:"role,omitempty"` +} + +type ButtonInfo struct { + Text string `json:"text,omitempty"` + Type string `json:"type,omitempty"` + ID string `json:"id,omitempty"` + AriaLabel string `json:"aria_label,omitempty"` + Disabled bool `json:"disabled"` +} + +// NetworkResult holds captured network analysis. +type NetworkResult struct { + TotalRequests int `json:"total_requests"` + APIEndpoints []APIEndpoint `json:"api_endpoints"` + StaticFiltered int `json:"static_filtered"` +} + +type APIEndpoint struct { + Method string `json:"method"` + URL string `json:"url"` + HasAuthHeader bool `json:"has_auth_header"` + AuthHeaders []string `json:"auth_headers,omitempty"` + ContentType string `json:"content_type,omitempty"` + Status int `json:"status"` +} + +// PatternResult holds the detected CLI pattern recommendation. +type PatternResult struct { + Type string `json:"type"` // dom-scrape, api-reverse, form-submit, async-poll + Confidence float64 `json:"confidence"` + Reason string `json:"reason"` +} + +// Run executes the full probe pipeline. +func Run(client *browser.Client, targetURL string) (*SiteProfile, error) { + profile := &SiteProfile{ + URL: targetURL, + Timestamp: time.Now().Format(time.RFC3339), + } + + // ── Phase 1: Start network capture BEFORE navigate ── + if err := client.NetworkStart(); err != nil { + profile.Warnings = append(profile.Warnings, fmt.Sprintf("network capture start failed: %v", err)) + } + + // ── Phase 2: Navigate (triggers page-load requests) ── + if err := client.Navigate(targetURL); err != nil { + return nil, fmt.Errorf("navigate failed: %w", err) + } + // Wait for page + async requests to settle + time.Sleep(3 * time.Second) + + // ── Phase 3: Auth detection ── + auth, warnings := DetectAuth(client) + profile.Auth = auth + profile.Warnings = append(profile.Warnings, warnings...) + + // ── Phase 4: DOM extraction ── + dom, domWarnings := ExtractDOM(client) + profile.DOM = dom + profile.Warnings = append(profile.Warnings, domWarnings...) + + // ── Phase 5: Stop network capture + analyze ── + if err := client.NetworkStop(); err != nil { + profile.Warnings = append(profile.Warnings, fmt.Sprintf("network stop failed: %v", err)) + } + network, netWarnings := AnalyzeNetwork(client) + profile.Network = network + profile.Warnings = append(profile.Warnings, netWarnings...) + + // ── Phase 6: Pattern detection ── + profile.Pattern = DetectPattern(profile.Auth, profile.DOM, profile.Network) + + return profile, nil +} + +// RunQuick runs a fast probe without network capture (DOM + auth only). +func RunQuick(client *browser.Client, targetURL string) (*SiteProfile, error) { + profile := &SiteProfile{ + URL: targetURL, + Timestamp: time.Now().Format(time.RFC3339), + } + + // Navigate + if err := client.Navigate(targetURL); err != nil { + return nil, fmt.Errorf("navigate failed: %w", err) + } + time.Sleep(2 * time.Second) + + // Auth + auth, warnings := DetectAuth(client) + profile.Auth = auth + profile.Warnings = append(profile.Warnings, warnings...) + + // DOM + dom, domWarnings := ExtractDOM(client) + profile.DOM = dom + profile.Warnings = append(profile.Warnings, domWarnings...) + + // Pattern (limited without network data) + profile.Pattern = DetectPattern(profile.Auth, profile.DOM, &NetworkResult{}) + + return profile, nil +}