From 9b45fe6948d703662bb00dc416c75206289c3385 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Fri, 4 Sep 2026 18:14:07 +0200 Subject: [PATCH 1/6] feat(sdk): add session.fetch for in-page HTTP requests --- docs/src/docs.json | 1 + .../features/sessions/browser-controls.mdx | 20 +++ docs/src/llms.txt | 1 + docs/src/sdk-reference/misc/fetchresponse.mdx | 51 +++++++ docs/src/sdk-reference/misc/remotesession.mdx | 14 ++ .../src/sdk-reference/remotesession/fetch.mdx | 52 +++++++ .../src/sdk-reference/remotesession/index.mdx | 18 +++ docs/src/snippets/browser-controls/fetch.mdx | 15 ++ docs/src/testers/browser-controls/fetch.py | 11 ++ .../src/notte_browser/session.py | 52 ++++++- .../notte-core/src/notte_core/data/fetch.py | 138 ++++++++++++++++++ .../src/notte_core/errors/actions.py | 23 +++ .../src/notte_sdk/endpoints/sessions.py | 39 ++++- tests/sdk/test_fetch_helper.py | 124 ++++++++++++++++ tests/test_fetch_helper.py | 43 ++++++ 15 files changed, 600 insertions(+), 2 deletions(-) create mode 100644 docs/src/sdk-reference/misc/fetchresponse.mdx create mode 100644 docs/src/sdk-reference/remotesession/fetch.mdx create mode 100644 docs/src/snippets/browser-controls/fetch.mdx create mode 100644 docs/src/testers/browser-controls/fetch.py create mode 100644 packages/notte-core/src/notte_core/data/fetch.py create mode 100644 tests/sdk/test_fetch_helper.py create mode 100644 tests/test_fetch_helper.py diff --git a/docs/src/docs.json b/docs/src/docs.json index 2acbf7798..a149c1ba6 100644 --- a/docs/src/docs.json +++ b/docs/src/docs.json @@ -249,6 +249,7 @@ "sdk-reference/remotesession/observe", "sdk-reference/remotesession/execute", "sdk-reference/remotesession/scrape", +"sdk-reference/remotesession/fetch", "sdk-reference/remotesession/replay", "sdk-reference/remotesession/cdp_url", "sdk-reference/remotesession/set_cookies", diff --git a/docs/src/features/sessions/browser-controls.mdx b/docs/src/features/sessions/browser-controls.mdx index 5e6eba224..bcd19a478 100644 --- a/docs/src/features/sessions/browser-controls.mdx +++ b/docs/src/features/sessions/browser-controls.mdx @@ -22,6 +22,7 @@ import Click from "/snippets/browser-controls/click.mdx"; import Fill from "/snippets/browser-controls/fill.mdx"; import Check from "/snippets/browser-controls/check.mdx"; import EvaluateJs from "/snippets/browser-controls/eval_js.mdx"; +import Fetch from "/snippets/browser-controls/fetch.mdx"; import SelectDropdownOption from "/snippets/browser-controls/select_dropdown_option.mdx"; import PressKey from "/snippets/browser-controls/press_key.mdx"; import ScrollUp from "/snippets/browser-controls/scroll_up.mdx"; @@ -205,6 +206,25 @@ Evaluate JavaScript code on the current page and return the result. `session.eva --- +### Fetch + +Issue an HTTP request from the page the session is on. `session.fetch(url)` runs the browser's own `fetch()` inside the current page, so the request carries the page's cookies, the session's proxy and the browser's network fingerprint. A relative URL resolves against the current page, which also keeps it same-origin; a cross-origin URL is subject to CORS as in any browser tab, so `goto` the target origin first. The response is shaped like `requests`: `status_code`, `ok`, `headers`, `text`, `url`, `json()` and `raise_for_status()`. + + + +**Parameters:** +- `url` (str): The URL to request, absolute or relative to the current page +- `method` (str): HTTP method, `GET` by default +- `headers` (dict): Extra request headers +- `params` (dict): Query parameters appended to the URL +- `json` (Any): Body serialised as JSON with an `application/json` content type +- `data` (str | dict): Body sent verbatim, or form-encoded when a dict +- `timeout` (float): Seconds before the request is aborted + +**Use for:** Calling a site's own JSON endpoints with the session's cookies and IP, without leaving the browser + +--- + ## Scrolling Actions ### ScrollUp diff --git a/docs/src/llms.txt b/docs/src/llms.txt index 7410c7505..8393884e1 100644 --- a/docs/src/llms.txt +++ b/docs/src/llms.txt @@ -368,6 +368,7 @@ The SDK docs below are for generated-code editing and reference. They are not th - [observe](https://docs.notte.cc/sdk-reference/remotesession/observe.md): Observes the current session page - [execute](https://docs.notte.cc/sdk-reference/remotesession/execute.md): Executes an action on the current session page - [scrape](https://docs.notte.cc/sdk-reference/remotesession/scrape.md): Scrape the current page data +- [fetch](https://docs.notte.cc/sdk-reference/remotesession/fetch.md): Issue an HTTP request from the page the session is on and return the response - [replay](https://docs.notte.cc/sdk-reference/remotesession/replay.md): Get presigned URLs for the session replay - [cdp_url](https://docs.notte.cc/sdk-reference/remotesession/cdp_url.md): Get the Chrome DevTools Protocol WebSocket URL for the session - [set_cookies](https://docs.notte.cc/sdk-reference/remotesession/set_cookies.md): Uploads cookies to the session diff --git a/docs/src/sdk-reference/misc/fetchresponse.mdx b/docs/src/sdk-reference/misc/fetchresponse.mdx new file mode 100644 index 000000000..4ea93ab20 --- /dev/null +++ b/docs/src/sdk-reference/misc/fetchresponse.mdx @@ -0,0 +1,51 @@ +--- +title: "FetchResponse" +description: "The response of a fetch call, with the shape of a requests response" +--- + + + +A non-2xx status is a response, not an error; call `raise_for_status()` +for the `requests` behaviour. `url` is the final URL after redirects + +## Methods + +### from_evaluated + +```python +from_evaluated(raw: str) -> FetchResponse +``` + +Read the envelope `build_fetch_script` returns from the evaluated string + +**Returns:** + +[`FetchResponse`](/sdk-reference/misc/fetchresponse)[`FetchResponse`](/sdk-reference/misc/fetchresponse.md) + +--- + +### json + +```python +json() -> Any +``` + +**Returns:** + +`Any` + +--- + +### raise_for_status + +```python +raise_for_status() -> None +``` + +--- + + + +## Module + +`notte_core.data.fetch` diff --git a/docs/src/sdk-reference/misc/remotesession.mdx b/docs/src/sdk-reference/misc/remotesession.mdx index c0126d86c..d967e6802 100644 --- a/docs/src/sdk-reference/misc/remotesession.mdx +++ b/docs/src/sdk-reference/misc/remotesession.mdx @@ -89,6 +89,20 @@ Result containing execution details, any errors, and the updated session state. --- +### fetch + +```python +fetch(url: , method: = GET, headers: collections.abc.Mapping[str, str] | None = None, params: collections.abc.Mapping[str, typing.Any] | None = None, json: typing.Any = None, data: str | collections.abc.Mapping[str, typing.Any] | None = None, timeout: float | None = None) -> +``` + +Issue an HTTP request from the page the session is on and return the response + +**Returns:** + +[`FetchResponse`](/sdk-reference/misc/fetchresponse)[`FetchResponse`](/sdk-reference/misc/fetchresponse.md) + +--- + ### get_cookies ```python diff --git a/docs/src/sdk-reference/remotesession/fetch.mdx b/docs/src/sdk-reference/remotesession/fetch.mdx new file mode 100644 index 000000000..0d429b6c8 --- /dev/null +++ b/docs/src/sdk-reference/remotesession/fetch.mdx @@ -0,0 +1,52 @@ +--- +title: "fetch" +description: "Issue an HTTP request from the page the session is on and return the response" +--- +import AgentMdNotice from '/partials/agent-md-notice.mdx'; + + + +The request runs inside the browser through `fetch()`, so it carries the +page's cookies, the session's proxy and the browser's own network +fingerprint. A relative `url` resolves against the current page, which +also makes it same-origin; a cross-origin URL is subject to CORS exactly +as in a browser tab, so `goto` the target origin first. Redirects are +followed and the final URL is on `response.url`. A non-2xx status is +returned, not raised; call `response.raise_for_status()` for the +`requests` behaviour. A network failure surfaces as the JavaScript error. + +`json` is serialised as the body with an `application/json` content type, +`data` as a form body when it is a mapping or verbatim when it is a string. + +```python +session.execute(type="goto", url="https://en.wikipedia.org/wiki/Main_Page") +summary = session.fetch("/api/rest_v1/page/summary/Main_Page").json() +``` + + +## Parameters + + + + + + + + + + + + + + + + + + + + + + +## Returns + +[`FetchResponse`](/sdk-reference/misc/fetchresponse)[`FetchResponse`](/sdk-reference/misc/fetchresponse.md) diff --git a/docs/src/sdk-reference/remotesession/index.mdx b/docs/src/sdk-reference/remotesession/index.mdx index 3d92d99ea..4dbcd52e8 100644 --- a/docs/src/sdk-reference/remotesession/index.mdx +++ b/docs/src/sdk-reference/remotesession/index.mdx @@ -109,6 +109,24 @@ Attributes: Executes an action on the current session page + + + Issue an HTTP request from the page the session is on and return the response + + + + + Issue an HTTP request from the page the session is on and return the response + + str | Exec """ return asyncio.run(self.aevaluate_js(code, raise_on_failure=raise_on_failure)) + async def afetch( + self, + url: str, + *, + method: str = "GET", + headers: Mapping[str, str] | None = None, + params: Mapping[str, Any] | None = None, + json: Any = None, + data: FetchData | None = None, + timeout: float | None = None, + ) -> FetchResponse: + """ + Issue an HTTP request from the page the session is on and return the response. + + The request runs inside the browser through `fetch()`, so it carries the + page's cookies, the session's proxy and the browser's own network + fingerprint. A relative `url` resolves against the current page, which + also makes it same-origin; a cross-origin URL is subject to CORS exactly + as in a browser tab, so `goto` the target origin first. Redirects are + followed and the final URL is on `response.url`. A non-2xx status is + returned, not raised; call `response.raise_for_status()` for the + `requests` behaviour. A network failure surfaces as the JavaScript error. + + `json` is serialised as the body with an `application/json` content type, + `data` as a form body when it is a mapping or verbatim when it is a string. + """ + script = build_fetch_script( + url, method=method, headers=headers, params=params, json_body=json, data=data, timeout=timeout + ) + return FetchResponse.from_evaluated(await self.aevaluate_js(script)) + + def fetch( + self, + url: str, + *, + method: str = "GET", + headers: Mapping[str, str] | None = None, + params: Mapping[str, Any] | None = None, + json: Any = None, + data: FetchData | None = None, + timeout: float | None = None, + ) -> FetchResponse: + """ + Synchronous version of afetch. + """ + return asyncio.run( + self.afetch(url, method=method, headers=headers, params=params, json=json, data=data, timeout=timeout) + ) + @overload async def ascrape(self, /, *, only_images: Literal[True], raise_on_failure: bool = True) -> list[ImageData]: ... diff --git a/packages/notte-core/src/notte_core/data/fetch.py b/packages/notte-core/src/notte_core/data/fetch.py new file mode 100644 index 000000000..f8b6e0d2e --- /dev/null +++ b/packages/notte-core/src/notte_core/data/fetch.py @@ -0,0 +1,138 @@ +"""Issue an HTTP request from the page a session is on. + +`session.fetch()` runs the browser's own `fetch()` inside the current page, so +the request carries the page's cookies, the session's proxy and the browser's +network fingerprint. This module builds the script and reads the result back; +it is shared by the remote SDK session and the local browser session. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, cast +from urllib.parse import urlencode + +from notte_core.errors.actions import FetchResponseDecodeError, FetchStatusError + +FetchData = str | Mapping[str, Any] + +_CONTENT_TYPE = "content-type" + + +def _has_header(headers: Mapping[str, str], name: str) -> bool: + return any(key.lower() == name for key in headers) + + +def build_fetch_script( + url: str, + *, + method: str = "GET", + headers: Mapping[str, str] | None = None, + params: Mapping[str, Any] | None = None, + json_body: Any = None, + data: FetchData | None = None, + timeout: float | None = None, +) -> str: + """Return the JavaScript that performs the request and serialises the response. + + The script is an async IIFE, which `evaluate_js` awaits. It returns a JSON + string so the value survives the evaluate round-trip unchanged. + """ + if json_body is not None and data is not None: + raise ValueError("pass either json or data, not both") + if timeout is not None and timeout <= 0: + raise ValueError("timeout must be positive") + + request_url = url + if params: + separator = "&" if "?" in url else "?" + request_url = f"{url}{separator}{urlencode(params, doseq=True)}" + + request_headers: dict[str, str] = dict(headers or {}) + body: str | None = None + if json_body is not None: + body = json.dumps(json_body) + if not _has_header(request_headers, _CONTENT_TYPE): + request_headers["Content-Type"] = "application/json" + elif isinstance(data, Mapping): + body = urlencode(data, doseq=True) + if not _has_header(request_headers, _CONTENT_TYPE): + request_headers["Content-Type"] = "application/x-www-form-urlencoded" + elif data is not None: + body = data + + init: dict[str, Any] = { + "method": method.upper(), + "headers": request_headers, + "credentials": "include", + "redirect": "follow", + } + if body is not None: + init["body"] = body + + abort = "" + if timeout is not None: + abort = ( + "const controller = new AbortController();" + f"setTimeout(() => controller.abort(), {int(timeout * 1000)});" + "init.signal = controller.signal;" + ) + return ( + "(async () => {" + f"const init = {json.dumps(init)};" + f"{abort}" + f"const response = await fetch({json.dumps(request_url)}, init);" + "const text = await response.text();" + "const headers = {};" + "response.headers.forEach((value, key) => { headers[key] = value; });" + "return JSON.stringify({status: response.status, url: response.url, headers: headers, text: text});" + "})()" + ) + + +@dataclass(frozen=True) +class FetchResponse: + """The response of a fetch call, with the shape of a requests response. + + A non-2xx status is a response, not an error; call `raise_for_status()` + for the `requests` behaviour. `url` is the final URL after redirects. + """ + + status_code: int + headers: dict[str, str] + text: str + url: str + + @property + def ok(self) -> bool: + return self.status_code < 400 + + def json(self) -> Any: + return json.loads(self.text) + + def raise_for_status(self) -> None: + if self.status_code >= 400: + raise FetchStatusError(status_code=self.status_code, url=self.url) + + @classmethod + def from_evaluated(cls, raw: str) -> FetchResponse: + """Read the envelope `build_fetch_script` returns from the evaluated string.""" + try: + payload: Any = json.loads(raw) + except json.JSONDecodeError as exc: + raise FetchResponseDecodeError(reason=str(exc)) from exc + if not isinstance(payload, dict): + raise FetchResponseDecodeError(reason="envelope is not an object") + envelope = cast(dict[str, Any], payload) + try: + raw_headers: Any = envelope.get("headers") or {} + return cls( + status_code=int(envelope["status"]), + headers={str(key): str(value) for key, value in dict(raw_headers).items()}, + text=str(envelope.get("text", "")), + url=str(envelope.get("url", "")), + ) + except (KeyError, TypeError, ValueError) as exc: + raise FetchResponseDecodeError(reason=str(exc)) from exc diff --git a/packages/notte-core/src/notte_core/errors/actions.py b/packages/notte-core/src/notte_core/errors/actions.py index 60d7f81ad..ae972c055 100644 --- a/packages/notte-core/src/notte_core/errors/actions.py +++ b/packages/notte-core/src/notte_core/errors/actions.py @@ -32,6 +32,29 @@ def __init__(self) -> None: ) +class FetchStatusError(ActionError): + def __init__(self, status_code: int, url: str) -> None: + self.status_code: int = status_code + self.url: str = url + message = f"fetch of {url} returned HTTP {status_code}" + super().__init__( + dev_message=message, + user_message=f"{message}.", + agent_message=message, + ) + + +class FetchResponseDecodeError(ActionError): + def __init__(self, reason: str) -> None: + message = f"fetch script returned an unreadable response envelope: {reason}" + super().__init__( + dev_message=message, + user_message=f"{message}.", + agent_message=message, + should_notify_team=True, + ) + + class NotEnoughActionsListedError(ActionError): def __init__(self, n_trials: int, n_actions: int, threshold: float) -> None: super().__init__( diff --git a/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py b/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py index 2bfc3fec4..e30af3cfa 100644 --- a/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py +++ b/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py @@ -1,5 +1,5 @@ import time -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from enum import StrEnum from pathlib import Path from types import TracebackType @@ -43,6 +43,7 @@ from notte_core.common.logging import logger from notte_core.common.resource import SyncResource from notte_core.common.telemetry import track_usage +from notte_core.data.fetch import FetchData, FetchResponse, build_fetch_script from notte_core.data.space import ImageData, StructuredData, TBaseModel from notte_core.errors.actions import EvaluateJsNoDataError from notte_core.errors.base import NotteBaseError @@ -1636,3 +1637,39 @@ def evaluate_js(self, code: str, *, raise_on_failure: bool = True) -> str | Exec # an API build that predates the eval-js fix reports success without data raise EvaluateJsNoDataError() return result.data.markdown + + def fetch( + self, + url: str, + *, + method: str = "GET", + headers: Mapping[str, str] | None = None, + params: Mapping[str, Any] | None = None, + json: Any = None, + data: FetchData | None = None, + timeout: float | None = None, + ) -> FetchResponse: + """ + Issue an HTTP request from the page the session is on and return the response. + + The request runs inside the browser through `fetch()`, so it carries the + page's cookies, the session's proxy and the browser's own network + fingerprint. A relative `url` resolves against the current page, which + also makes it same-origin; a cross-origin URL is subject to CORS exactly + as in a browser tab, so `goto` the target origin first. Redirects are + followed and the final URL is on `response.url`. A non-2xx status is + returned, not raised; call `response.raise_for_status()` for the + `requests` behaviour. A network failure surfaces as the JavaScript error. + + `json` is serialised as the body with an `application/json` content type, + `data` as a form body when it is a mapping or verbatim when it is a string. + + ```python + session.execute(type="goto", url="https://en.wikipedia.org/wiki/Main_Page") + summary = session.fetch("/api/rest_v1/page/summary/Main_Page").json() + ``` + """ + script = build_fetch_script( + url, method=method, headers=headers, params=params, json_body=json, data=data, timeout=timeout + ) + return FetchResponse.from_evaluated(self.evaluate_js(script)) diff --git a/tests/sdk/test_fetch_helper.py b/tests/sdk/test_fetch_helper.py new file mode 100644 index 000000000..f87c2c363 --- /dev/null +++ b/tests/sdk/test_fetch_helper.py @@ -0,0 +1,124 @@ +"""Remote `fetch()`: the request runs in the page via `evaluate_js` and comes back requests-shaped.""" + +import datetime as dt +import json + +import pytest +from notte_core.actions import EvaluateJsAction +from notte_core.browser.observation import ExecutionResult +from notte_core.data.fetch import FetchResponse, build_fetch_script +from notte_core.data.space import DataSpace +from notte_core.errors.actions import FetchResponseDecodeError, FetchStatusError + +from tests.sdk.test_execute_raise_on_failure import over_the_wire, remote_session + + +def envelope(status: int = 200, text: str = '{"ok": true}', url: str = "https://example.com/api") -> str: + return json.dumps({"status": status, "url": url, "headers": {"content-type": "application/json"}, "text": text}) + + +def eval_result(markdown: str) -> ExecutionResult: + now = dt.datetime.now(dt.timezone.utc) + return ExecutionResult( + action=EvaluateJsAction(code="fetch"), + success=True, + message="ok", + data=DataSpace(markdown=markdown), + started_at=now, + ended_at=now, + ) + + +# --- the script ----------------------------------------------------------------- + + +def test_script_defaults_to_a_credentialed_get_with_no_body() -> None: + script = build_fetch_script("/api") + + assert script.startswith("(async () => {") + assert '"method": "GET"' in script + assert '"credentials": "include"' in script + assert 'fetch("/api", init)' in script + assert '"body"' not in script + assert "AbortController" not in script + + +def test_script_appends_params_to_the_query_string() -> None: + assert 'fetch("/api?page=2&q=a+b", init)' in build_fetch_script("/api", params={"page": 2, "q": "a b"}) + assert 'fetch("/api?x=1&page=2", init)' in build_fetch_script("/api?x=1", params={"page": 2}) + + +def test_script_serialises_a_json_body_and_sets_the_content_type() -> None: + script = build_fetch_script("/graphql", method="post", json_body={"query": "{ me }"}) + + assert '"method": "POST"' in script + assert '"Content-Type": "application/json"' in script + assert json.dumps(json.dumps({"query": "{ me }"})) in script + + +def test_script_keeps_a_caller_content_type() -> None: + script = build_fetch_script("/x", json_body={}, headers={"content-type": "application/graphql-response+json"}) + + assert script.count("ontent-") == 1 + assert "application/graphql-response+json" in script + + +def test_script_form_encodes_a_mapping_and_passes_a_string_through() -> None: + form = build_fetch_script("/login", method="POST", data={"user": "a b", "pw": "c"}) + assert '"body": "user=a+b&pw=c"' in form + assert '"Content-Type": "application/x-www-form-urlencoded"' in form + + raw = build_fetch_script("/raw", method="PUT", data="") + assert '"body": ""' in raw + assert "Content-Type" not in raw + + +def test_script_rejects_json_and_data_together() -> None: + with pytest.raises(ValueError, match="either json or data"): + _ = build_fetch_script("/x", json_body={}, data="y") + + +def test_script_aborts_after_the_timeout() -> None: + script = build_fetch_script("/slow", timeout=2.5) + + assert "controller.abort(), 2500" in script + assert "init.signal = controller.signal" in script + with pytest.raises(ValueError, match="positive"): + _ = build_fetch_script("/slow", timeout=0) + + +# --- the response ---------------------------------------------------------------- + + +def test_fetch_returns_a_requests_shaped_response() -> None: + session = remote_session(over_the_wire(eval_result(envelope()))) + + response = session.fetch("/api") + + assert isinstance(response, FetchResponse) + assert response.status_code == 200 + assert response.ok + assert response.json() == {"ok": True} + assert response.headers["content-type"] == "application/json" + assert response.url == "https://example.com/api" + response.raise_for_status() + + +def test_fetch_returns_http_errors_and_raises_only_when_asked() -> None: + session = remote_session(over_the_wire(eval_result(envelope(status=403, text="denied")))) + + response = session.fetch("/api") + + assert response.status_code == 403 + assert not response.ok + assert response.text == "denied" + with pytest.raises(FetchStatusError, match="HTTP 403") as raised: + response.raise_for_status() + assert raised.value.status_code == 403 + + +def test_fetch_rejects_an_unreadable_envelope() -> None: + session = remote_session(over_the_wire(eval_result("not json"))) + + with pytest.raises(FetchResponseDecodeError, match="unreadable"): + _ = session.fetch("/api") diff --git a/tests/test_fetch_helper.py b/tests/test_fetch_helper.py new file mode 100644 index 000000000..a6c95b3af --- /dev/null +++ b/tests/test_fetch_helper.py @@ -0,0 +1,43 @@ +"""`afetch()` runs the browser's `fetch()` inside the page and returns the response.""" + +import pytest +from notte_browser.session import NotteSession +from notte_core.errors.actions import ActionExecutionError + + +@pytest.mark.asyncio +async def test_afetch_resolves_a_relative_url_against_the_page() -> None: + async with NotteSession(headless=True) as session: + _ = await session.aexecute(type="goto", url="https://www.example.com/") + + response = await session.afetch("/") + + assert response.status_code == 200 + assert response.ok + assert "Example Domain" in response.text + assert response.url.startswith("https://www.example.com/") + assert "content-type" in response.headers + + +@pytest.mark.asyncio +async def test_afetch_json_reads_the_body() -> None: + async with NotteSession(headless=True) as session: + _ = await session.aexecute(type="goto", url="https://www.example.com/") + + response = await session.afetch("data:application/json,%7B%22a%22%3A1%7D") + + assert response.json() == {"a": 1} + + +@pytest.mark.asyncio +async def test_afetch_network_failure_raises_the_js_error() -> None: + async with NotteSession(headless=True) as session: + _ = await session.aexecute(type="goto", url="https://www.example.com/") + + with pytest.raises(ActionExecutionError, match="JavaScript evaluation failed"): + _ = await session.afetch("https://nonexistent.invalid/") + + +# NOTE: no sync-variant test here on purpose, for the same reason as +# test_evaluate_js_helper.py: a sync NotteSession in this process breaks the +# next async browser launch. The sync wrapper is a one-line delegation to afetch. From 677102deed003ff80985e99c4f6783c52b72b0a3 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Fri, 4 Sep 2026 18:31:58 +0200 Subject: [PATCH 2/6] fix(fetch): keep params before fragments, reject GET/HEAD bodies --- packages/notte-core/src/notte_core/data/fetch.py | 14 +++++++++++--- tests/sdk/test_fetch_helper.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/notte-core/src/notte_core/data/fetch.py b/packages/notte-core/src/notte_core/data/fetch.py index f8b6e0d2e..42ae62cf8 100644 --- a/packages/notte-core/src/notte_core/data/fetch.py +++ b/packages/notte-core/src/notte_core/data/fetch.py @@ -12,7 +12,7 @@ from collections.abc import Mapping from dataclasses import dataclass from typing import Any, cast -from urllib.parse import urlencode +from urllib.parse import urlencode, urlsplit, urlunsplit from notte_core.errors.actions import FetchResponseDecodeError, FetchStatusError @@ -47,8 +47,13 @@ def build_fetch_script( request_url = url if params: - separator = "&" if "?" in url else "?" - request_url = f"{url}{separator}{urlencode(params, doseq=True)}" + # insert before any fragment: the browser strips `#...` before sending, + # so parameters appended after it would be silently dropped + parts = urlsplit(url) + query = urlencode(params, doseq=True) + if parts.query: + query = f"{parts.query}&{query}" + request_url = urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment)) request_headers: dict[str, str] = dict(headers or {}) body: str | None = None @@ -70,6 +75,9 @@ def build_fetch_script( "redirect": "follow", } if body is not None: + if init["method"] in {"GET", "HEAD"}: + # browser fetch() rejects these outright, so fail before the round-trip + raise ValueError(f"{init['method']} requests cannot have a body") init["body"] = body abort = "" diff --git a/tests/sdk/test_fetch_helper.py b/tests/sdk/test_fetch_helper.py index f87c2c363..4c7fe9186 100644 --- a/tests/sdk/test_fetch_helper.py +++ b/tests/sdk/test_fetch_helper.py @@ -46,6 +46,11 @@ def test_script_defaults_to_a_credentialed_get_with_no_body() -> None: def test_script_appends_params_to_the_query_string() -> None: assert 'fetch("/api?page=2&q=a+b", init)' in build_fetch_script("/api", params={"page": 2, "q": "a b"}) assert 'fetch("/api?x=1&page=2", init)' in build_fetch_script("/api?x=1", params={"page": 2}) + # before the fragment, which the browser strips before sending + assert 'fetch("/items?page=2#results", init)' in build_fetch_script("/items#results", params={"page": 2}) + assert 'fetch("https://a.test/p?x=1&page=2#f", init)' in build_fetch_script( + "https://a.test/p?x=1#f", params={"page": 2} + ) def test_script_serialises_a_json_body_and_sets_the_content_type() -> None: @@ -78,6 +83,13 @@ def test_script_rejects_json_and_data_together() -> None: _ = build_fetch_script("/x", json_body={}, data="y") +def test_script_rejects_a_body_on_get_and_head() -> None: + with pytest.raises(ValueError, match="GET requests cannot have a body"): + _ = build_fetch_script("/x", json_body={"a": 1}) + with pytest.raises(ValueError, match="HEAD requests cannot have a body"): + _ = build_fetch_script("/x", method="head", data="y") + + def test_script_aborts_after_the_timeout() -> None: script = build_fetch_script("/slow", timeout=2.5) From 307a063982817553a039be443de9102079a34989 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Fri, 4 Sep 2026 18:32:36 +0200 Subject: [PATCH 3/6] test(fetch): send the content-type case as a POST --- tests/sdk/test_fetch_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/sdk/test_fetch_helper.py b/tests/sdk/test_fetch_helper.py index 4c7fe9186..0bb275322 100644 --- a/tests/sdk/test_fetch_helper.py +++ b/tests/sdk/test_fetch_helper.py @@ -62,7 +62,9 @@ def test_script_serialises_a_json_body_and_sets_the_content_type() -> None: def test_script_keeps_a_caller_content_type() -> None: - script = build_fetch_script("/x", json_body={}, headers={"content-type": "application/graphql-response+json"}) + script = build_fetch_script( + "/x", method="POST", json_body={}, headers={"content-type": "application/graphql-response+json"} + ) assert script.count("ontent-") == 1 assert "application/graphql-response+json" in script From a614c686748a4cd97e040e58f571898b2b982568 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Sat, 5 Sep 2026 19:50:16 +0200 Subject: [PATCH 4/6] feat(fetch): return a requests.Response instead of a custom type --- .../features/sessions/browser-controls.mdx | 2 +- docs/src/sdk-reference/misc/fetchresponse.mdx | 51 ----------- docs/src/sdk-reference/misc/remotesession.mdx | 4 +- docs/src/sdk-reference/misc/response.mdx | 65 ++++++++++++++ .../src/sdk-reference/remotesession/fetch.mdx | 9 +- .../src/notte_browser/session.py | 16 ++-- .../notte-core/src/notte_core/data/fetch.py | 88 +++++++++---------- .../src/notte_core/errors/actions.py | 12 --- .../src/notte_sdk/endpoints/sessions.py | 14 +-- tests/sdk/test_fetch_helper.py | 21 +++-- 10 files changed, 145 insertions(+), 137 deletions(-) delete mode 100644 docs/src/sdk-reference/misc/fetchresponse.mdx create mode 100644 docs/src/sdk-reference/misc/response.mdx diff --git a/docs/src/features/sessions/browser-controls.mdx b/docs/src/features/sessions/browser-controls.mdx index bcd19a478..4177a19bb 100644 --- a/docs/src/features/sessions/browser-controls.mdx +++ b/docs/src/features/sessions/browser-controls.mdx @@ -208,7 +208,7 @@ Evaluate JavaScript code on the current page and return the result. `session.eva ### Fetch -Issue an HTTP request from the page the session is on. `session.fetch(url)` runs the browser's own `fetch()` inside the current page, so the request carries the page's cookies, the session's proxy and the browser's network fingerprint. A relative URL resolves against the current page, which also keeps it same-origin; a cross-origin URL is subject to CORS as in any browser tab, so `goto` the target origin first. The response is shaped like `requests`: `status_code`, `ok`, `headers`, `text`, `url`, `json()` and `raise_for_status()`. +Issue an HTTP request from the page the session is on. `session.fetch(url)` runs the browser's own `fetch()` inside the current page, so the request carries the page's cookies, the session's proxy and the browser's network fingerprint. A relative URL resolves against the current page, which also keeps it same-origin; a cross-origin URL is subject to CORS as in any browser tab, so `goto` the target origin first. The result is a standard `requests.Response`: `status_code`, `ok`, `headers`, `text`, `url`, `json()` and `raise_for_status()` work as usual. diff --git a/docs/src/sdk-reference/misc/fetchresponse.mdx b/docs/src/sdk-reference/misc/fetchresponse.mdx deleted file mode 100644 index 4ea93ab20..000000000 --- a/docs/src/sdk-reference/misc/fetchresponse.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: "FetchResponse" -description: "The response of a fetch call, with the shape of a requests response" ---- - - - -A non-2xx status is a response, not an error; call `raise_for_status()` -for the `requests` behaviour. `url` is the final URL after redirects - -## Methods - -### from_evaluated - -```python -from_evaluated(raw: str) -> FetchResponse -``` - -Read the envelope `build_fetch_script` returns from the evaluated string - -**Returns:** - -[`FetchResponse`](/sdk-reference/misc/fetchresponse)[`FetchResponse`](/sdk-reference/misc/fetchresponse.md) - ---- - -### json - -```python -json() -> Any -``` - -**Returns:** - -`Any` - ---- - -### raise_for_status - -```python -raise_for_status() -> None -``` - ---- - - - -## Module - -`notte_core.data.fetch` diff --git a/docs/src/sdk-reference/misc/remotesession.mdx b/docs/src/sdk-reference/misc/remotesession.mdx index d967e6802..f4ffeb30e 100644 --- a/docs/src/sdk-reference/misc/remotesession.mdx +++ b/docs/src/sdk-reference/misc/remotesession.mdx @@ -92,14 +92,14 @@ Result containing execution details, any errors, and the updated session state. ### fetch ```python -fetch(url: , method: = GET, headers: collections.abc.Mapping[str, str] | None = None, params: collections.abc.Mapping[str, typing.Any] | None = None, json: typing.Any = None, data: str | collections.abc.Mapping[str, typing.Any] | None = None, timeout: float | None = None) -> +fetch(url: , method: = GET, headers: collections.abc.Mapping[str, str] | None = None, params: collections.abc.Mapping[str, typing.Any] | None = None, json: typing.Any = None, data: str | collections.abc.Mapping[str, typing.Any] | None = None, timeout: float | None = None) -> ``` Issue an HTTP request from the page the session is on and return the response **Returns:** -[`FetchResponse`](/sdk-reference/misc/fetchresponse)[`FetchResponse`](/sdk-reference/misc/fetchresponse.md) +[`Response`](/sdk-reference/misc/response)[`Response`](/sdk-reference/misc/response.md) --- diff --git a/docs/src/sdk-reference/misc/response.mdx b/docs/src/sdk-reference/misc/response.mdx new file mode 100644 index 000000000..3e43554ab --- /dev/null +++ b/docs/src/sdk-reference/misc/response.mdx @@ -0,0 +1,65 @@ +--- +title: "Response" +description: "The :class:`Response ` object, which contains a" +--- + + +server's response to an HTTP request + +## Methods + +### close + +```python +close() +``` + +Releases the connection back to the pool + +--- + +### iter_content + +```python +iter_content(chunk_size = 1, decode_unicode = False) +``` + +Iterates over the response data + +--- + +### iter_lines + +```python +iter_lines(chunk_size = 512, decode_unicode = False, delimiter = None) +``` + +Iterates over the response data, one line at a time + +--- + +### json + +```python +json(kwargs) +``` + +Decodes the JSON response body (if any) as a Python object + +--- + +### raise_for_status + +```python +raise_for_status() +``` + +Raises :class:`HTTPError`, if one occurred + +--- + + + +## Module + +`requests.models` diff --git a/docs/src/sdk-reference/remotesession/fetch.mdx b/docs/src/sdk-reference/remotesession/fetch.mdx index 0d429b6c8..a8244b3d6 100644 --- a/docs/src/sdk-reference/remotesession/fetch.mdx +++ b/docs/src/sdk-reference/remotesession/fetch.mdx @@ -11,9 +11,10 @@ page's cookies, the session's proxy and the browser's own network fingerprint. A relative `url` resolves against the current page, which also makes it same-origin; a cross-origin URL is subject to CORS exactly as in a browser tab, so `goto` the target origin first. Redirects are -followed and the final URL is on `response.url`. A non-2xx status is -returned, not raised; call `response.raise_for_status()` for the -`requests` behaviour. A network failure surfaces as the JavaScript error. +followed and the final URL is on `response.url`. The result is a standard +`requests.Response`: a non-2xx status is returned, not raised, and +`response.raise_for_status()` raises `requests.HTTPError`. A network +failure surfaces as the JavaScript error. `json` is serialised as the body with an `application/json` content type, `data` as a form body when it is a mapping or verbatim when it is a string. @@ -49,4 +50,4 @@ summary = session.fetch("/api/rest_v1/page/summary/Main_Page").json() ## Returns -[`FetchResponse`](/sdk-reference/misc/fetchresponse)[`FetchResponse`](/sdk-reference/misc/fetchresponse.md) +[`Response`](/sdk-reference/misc/response)[`Response`](/sdk-reference/misc/response.md) diff --git a/packages/notte-browser/src/notte_browser/session.py b/packages/notte-browser/src/notte_browser/session.py index a18a89d20..2f9d00651 100644 --- a/packages/notte-browser/src/notte_browser/session.py +++ b/packages/notte-browser/src/notte_browser/session.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, ClassVar, Literal, Unpack, overload +import requests from litellm import BaseModel from notte_core import enable_nest_asyncio from notte_core.actions import ( @@ -63,7 +64,7 @@ from notte_core.common.resource import AsyncResource, SyncResource from notte_core.common.telemetry import track_usage from notte_core.credentials.base import BaseVault, LocatorAttributes -from notte_core.data.fetch import FetchData, FetchResponse, build_fetch_script +from notte_core.data.fetch import FetchData, build_fetch_script, response_from_evaluated from notte_core.data.space import DataSpace, ImageData, StructuredData, TBaseModel from notte_core.errors.actions import ActionExecutionError, EvaluateJsNoDataError, InvalidActionError from notte_core.errors.base import NotteBaseError @@ -1096,7 +1097,7 @@ async def afetch( json: Any = None, data: FetchData | None = None, timeout: float | None = None, - ) -> FetchResponse: + ) -> requests.Response: """ Issue an HTTP request from the page the session is on and return the response. @@ -1105,9 +1106,10 @@ async def afetch( fingerprint. A relative `url` resolves against the current page, which also makes it same-origin; a cross-origin URL is subject to CORS exactly as in a browser tab, so `goto` the target origin first. Redirects are - followed and the final URL is on `response.url`. A non-2xx status is - returned, not raised; call `response.raise_for_status()` for the - `requests` behaviour. A network failure surfaces as the JavaScript error. + followed and the final URL is on `response.url`. The result is a standard + `requests.Response`: a non-2xx status is returned, not raised, and + `response.raise_for_status()` raises `requests.HTTPError`. A network + failure surfaces as the JavaScript error. `json` is serialised as the body with an `application/json` content type, `data` as a form body when it is a mapping or verbatim when it is a string. @@ -1115,7 +1117,7 @@ async def afetch( script = build_fetch_script( url, method=method, headers=headers, params=params, json_body=json, data=data, timeout=timeout ) - return FetchResponse.from_evaluated(await self.aevaluate_js(script)) + return response_from_evaluated(await self.aevaluate_js(script)) def fetch( self, @@ -1127,7 +1129,7 @@ def fetch( json: Any = None, data: FetchData | None = None, timeout: float | None = None, - ) -> FetchResponse: + ) -> requests.Response: """ Synchronous version of afetch. """ diff --git a/packages/notte-core/src/notte_core/data/fetch.py b/packages/notte-core/src/notte_core/data/fetch.py index 42ae62cf8..1b834c644 100644 --- a/packages/notte-core/src/notte_core/data/fetch.py +++ b/packages/notte-core/src/notte_core/data/fetch.py @@ -2,19 +2,24 @@ `session.fetch()` runs the browser's own `fetch()` inside the current page, so the request carries the page's cookies, the session's proxy and the browser's -network fingerprint. This module builds the script and reads the result back; -it is shared by the remote SDK session and the local browser session. +network fingerprint. This module builds the script and reads the result back +into a standard `requests.Response`; it is shared by the remote SDK session and +the local browser session. """ from __future__ import annotations +import io import json from collections.abc import Mapping -from dataclasses import dataclass +from http import HTTPStatus from typing import Any, cast from urllib.parse import urlencode, urlsplit, urlunsplit -from notte_core.errors.actions import FetchResponseDecodeError, FetchStatusError +import requests +from requests.structures import CaseInsensitiveDict + +from notte_core.errors.actions import FetchResponseDecodeError FetchData = str | Mapping[str, Any] @@ -100,47 +105,38 @@ def build_fetch_script( ) -@dataclass(frozen=True) -class FetchResponse: - """The response of a fetch call, with the shape of a requests response. +def response_from_evaluated(raw: str) -> requests.Response: + """Turn the envelope `build_fetch_script` returns into a `requests.Response`. - A non-2xx status is a response, not an error; call `raise_for_status()` - for the `requests` behaviour. `url` is the final URL after redirects. + A non-2xx status is a response, not an error; `raise_for_status()` raises + `requests.HTTPError` as usual. `url` is the final URL after redirects, and + the body is exposed through `text`, `content` and `json()`. """ - - status_code: int - headers: dict[str, str] - text: str - url: str - - @property - def ok(self) -> bool: - return self.status_code < 400 - - def json(self) -> Any: - return json.loads(self.text) - - def raise_for_status(self) -> None: - if self.status_code >= 400: - raise FetchStatusError(status_code=self.status_code, url=self.url) - - @classmethod - def from_evaluated(cls, raw: str) -> FetchResponse: - """Read the envelope `build_fetch_script` returns from the evaluated string.""" - try: - payload: Any = json.loads(raw) - except json.JSONDecodeError as exc: - raise FetchResponseDecodeError(reason=str(exc)) from exc - if not isinstance(payload, dict): - raise FetchResponseDecodeError(reason="envelope is not an object") - envelope = cast(dict[str, Any], payload) - try: - raw_headers: Any = envelope.get("headers") or {} - return cls( - status_code=int(envelope["status"]), - headers={str(key): str(value) for key, value in dict(raw_headers).items()}, - text=str(envelope.get("text", "")), - url=str(envelope.get("url", "")), - ) - except (KeyError, TypeError, ValueError) as exc: - raise FetchResponseDecodeError(reason=str(exc)) from exc + try: + payload: Any = json.loads(raw) + except json.JSONDecodeError as exc: + raise FetchResponseDecodeError(reason=str(exc)) from exc + if not isinstance(payload, dict): + raise FetchResponseDecodeError(reason="envelope is not an object") + envelope = cast(dict[str, Any], payload) + try: + status_code = int(envelope["status"]) + raw_headers: Any = envelope.get("headers") or {} + headers = {str(key): str(value) for key, value in dict(raw_headers).items()} + text = str(envelope.get("text", "")) + url = str(envelope.get("url", "")) + except (KeyError, TypeError, ValueError) as exc: + raise FetchResponseDecodeError(reason=str(exc)) from exc + + response = requests.Response() + response.status_code = status_code + response.headers = CaseInsensitiveDict(headers) + # the browser already decoded the body; hand it back as utf-8 so `.text` round-trips + response.encoding = "utf-8" + response.raw = io.BytesIO(text.encode("utf-8")) + response.url = url + try: + response.reason = HTTPStatus(status_code).phrase + except ValueError: + response.reason = "" + return response diff --git a/packages/notte-core/src/notte_core/errors/actions.py b/packages/notte-core/src/notte_core/errors/actions.py index ae972c055..8ef8d3789 100644 --- a/packages/notte-core/src/notte_core/errors/actions.py +++ b/packages/notte-core/src/notte_core/errors/actions.py @@ -32,18 +32,6 @@ def __init__(self) -> None: ) -class FetchStatusError(ActionError): - def __init__(self, status_code: int, url: str) -> None: - self.status_code: int = status_code - self.url: str = url - message = f"fetch of {url} returned HTTP {status_code}" - super().__init__( - dev_message=message, - user_message=f"{message}.", - agent_message=message, - ) - - class FetchResponseDecodeError(ActionError): def __init__(self, reason: str) -> None: message = f"fetch script returned an unreadable response envelope: {reason}" diff --git a/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py b/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py index e30af3cfa..d7ce1619a 100644 --- a/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py +++ b/packages/notte-sdk/src/notte_sdk/endpoints/sessions.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Literal, Unpack, overload from webbrowser import open as open_browser +import requests from notte_core.actions import BaseAction, InteractionActionUnion from notte_core.actions.typedicts import ( CaptchaSolveActionDict, @@ -43,7 +44,7 @@ from notte_core.common.logging import logger from notte_core.common.resource import SyncResource from notte_core.common.telemetry import track_usage -from notte_core.data.fetch import FetchData, FetchResponse, build_fetch_script +from notte_core.data.fetch import FetchData, build_fetch_script, response_from_evaluated from notte_core.data.space import ImageData, StructuredData, TBaseModel from notte_core.errors.actions import EvaluateJsNoDataError from notte_core.errors.base import NotteBaseError @@ -1648,7 +1649,7 @@ def fetch( json: Any = None, data: FetchData | None = None, timeout: float | None = None, - ) -> FetchResponse: + ) -> requests.Response: """ Issue an HTTP request from the page the session is on and return the response. @@ -1657,9 +1658,10 @@ def fetch( fingerprint. A relative `url` resolves against the current page, which also makes it same-origin; a cross-origin URL is subject to CORS exactly as in a browser tab, so `goto` the target origin first. Redirects are - followed and the final URL is on `response.url`. A non-2xx status is - returned, not raised; call `response.raise_for_status()` for the - `requests` behaviour. A network failure surfaces as the JavaScript error. + followed and the final URL is on `response.url`. The result is a standard + `requests.Response`: a non-2xx status is returned, not raised, and + `response.raise_for_status()` raises `requests.HTTPError`. A network + failure surfaces as the JavaScript error. `json` is serialised as the body with an `application/json` content type, `data` as a form body when it is a mapping or verbatim when it is a string. @@ -1672,4 +1674,4 @@ def fetch( script = build_fetch_script( url, method=method, headers=headers, params=params, json_body=json, data=data, timeout=timeout ) - return FetchResponse.from_evaluated(self.evaluate_js(script)) + return response_from_evaluated(self.evaluate_js(script)) diff --git a/tests/sdk/test_fetch_helper.py b/tests/sdk/test_fetch_helper.py index 0bb275322..9debae85a 100644 --- a/tests/sdk/test_fetch_helper.py +++ b/tests/sdk/test_fetch_helper.py @@ -1,14 +1,15 @@ -"""Remote `fetch()`: the request runs in the page via `evaluate_js` and comes back requests-shaped.""" +"""Remote `fetch()`: the request runs in the page via `evaluate_js` and comes back as a `requests.Response`.""" import datetime as dt import json import pytest +import requests from notte_core.actions import EvaluateJsAction from notte_core.browser.observation import ExecutionResult -from notte_core.data.fetch import FetchResponse, build_fetch_script +from notte_core.data.fetch import build_fetch_script from notte_core.data.space import DataSpace -from notte_core.errors.actions import FetchResponseDecodeError, FetchStatusError +from notte_core.errors.actions import FetchResponseDecodeError from tests.sdk.test_execute_raise_on_failure import over_the_wire, remote_session @@ -104,16 +105,20 @@ def test_script_aborts_after_the_timeout() -> None: # --- the response ---------------------------------------------------------------- -def test_fetch_returns_a_requests_shaped_response() -> None: +def test_fetch_returns_a_requests_response() -> None: session = remote_session(over_the_wire(eval_result(envelope()))) response = session.fetch("/api") - assert isinstance(response, FetchResponse) + assert isinstance(response, requests.Response) assert response.status_code == 200 assert response.ok + assert response.reason == "OK" assert response.json() == {"ok": True} - assert response.headers["content-type"] == "application/json" + assert response.text == '{"ok": true}' + assert response.content == b'{"ok": true}' + # browsers lowercase header names; requests keeps the lookup case-insensitive + assert response.headers["Content-Type"] == "application/json" assert response.url == "https://example.com/api" response.raise_for_status() @@ -126,9 +131,9 @@ def test_fetch_returns_http_errors_and_raises_only_when_asked() -> None: assert response.status_code == 403 assert not response.ok assert response.text == "denied" - with pytest.raises(FetchStatusError, match="HTTP 403") as raised: + with pytest.raises(requests.HTTPError, match="403 Client Error: Forbidden") as raised: response.raise_for_status() - assert raised.value.status_code == 403 + assert raised.value.response is response def test_fetch_rejects_an_unreadable_envelope() -> None: From 418f3a191a9cb88e7f6315363092ba45c88e2eb1 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Sat, 5 Sep 2026 20:24:43 +0200 Subject: [PATCH 5/6] fix(docs): make llms.txt generation offline for pre-commit --- .github/workflows/refresh-llms.yml | 6 +- docs/src/llms.txt | 17 +++-- docs/src/scripts/generate_llms.py | 69 ++++++++++++++---- makefile | 6 ++ tests/test_generate_llms_offline.py | 108 ++++++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 26 deletions(-) create mode 100644 tests/test_generate_llms_offline.py diff --git a/.github/workflows/refresh-llms.yml b/.github/workflows/refresh-llms.yml index ff5171b44..86de393df 100644 --- a/.github/workflows/refresh-llms.yml +++ b/.github/workflows/refresh-llms.yml @@ -63,12 +63,12 @@ jobs: # The generator reports a failed spec fetch on stderr and still exits 0, # so stderr is captured to a file and inspected after the run. status=0 - make docs-llms >/tmp/generate-llms.out 2>/tmp/generate-llms.err || status=$? + make docs-llms-refresh >/tmp/generate-llms.out 2>/tmp/generate-llms.err || status=$? cat /tmp/generate-llms.out cat /tmp/generate-llms.err >&2 if [ "$status" -ne 0 ]; then - echo "::error::make docs-llms failed with exit code $status." + echo "::error::make docs-llms-refresh failed with exit code $status." exit "$status" fi @@ -106,7 +106,7 @@ jobs: DATE=$(date -u +%Y-%m-%d) TITLE="docs: refresh generated llms.txt" BODY=$(printf '%s\n\n%s\n\n%s\n\n%s\n' \ - "Automated daily regeneration of \`$TARGET\` via \`make docs-llms\`, which builds the API section from the live OpenAPI spec." \ + "Automated daily regeneration of \`$TARGET\` via \`make docs-llms-refresh\`, which builds the API section from the live OpenAPI spec." \ "Updated $DATE (UTC): $STAT" \ "No hand edits. The run fails instead of committing if the spec cannot be fetched or comes back incomplete." \ "Opened by .github/workflows/refresh-llms.yml. Merges automatically once the required checks pass.") diff --git a/docs/src/llms.txt b/docs/src/llms.txt index 8393884e1..93d8a8a44 100644 --- a/docs/src/llms.txt +++ b/docs/src/llms.txt @@ -203,6 +203,8 @@ The SDK docs below are for generated-code editing and reference. They are not th ## APIs + + ## Agents - [POST Agent Start](https://docs.notte.cc/api-reference/agents/agent-start.md) @@ -301,7 +303,10 @@ The SDK docs below are for generated-code editing and reference. They are not th ## Sessions +- [DELETE Delete Session File](https://docs.notte.cc/api-reference/sessions/delete-session-file.md) +- [GET Download Session File](https://docs.notte.cc/api-reference/sessions/download-session-file.md) - [GET Get Session Script](https://docs.notte.cc/api-reference/sessions/get-session-script.md) +- [GET List Session Files](https://docs.notte.cc/api-reference/sessions/list-session-files.md) - [GET List Sessions](https://docs.notte.cc/api-reference/sessions/list-sessions.md) - [POST Page Execute](https://docs.notte.cc/api-reference/sessions/page-execute.md) - [POST Page Observe](https://docs.notte.cc/api-reference/sessions/page-observe.md) @@ -316,15 +321,7 @@ The SDK docs below are for generated-code editing and reference. They are not th - [POST Session Start](https://docs.notte.cc/api-reference/sessions/session-start.md) - [GET Session Status](https://docs.notte.cc/api-reference/sessions/session-status.md) - [DELETE Session Stop](https://docs.notte.cc/api-reference/sessions/session-stop.md) - -## Storage - -- [GET File Download](https://docs.notte.cc/api-reference/storage/file-download.md) -- [GET File Download Uploaded File](https://docs.notte.cc/api-reference/storage/file-download-uploaded-file.md) -- [GET File List Downloads](https://docs.notte.cc/api-reference/storage/file-list-downloads.md) -- [GET File List Uploads](https://docs.notte.cc/api-reference/storage/file-list-uploads.md) -- [POST File Upload](https://docs.notte.cc/api-reference/storage/file-upload.md) -- [POST File Upload Downloaded File](https://docs.notte.cc/api-reference/storage/file-upload-downloaded-file.md) +- [POST Upload Session File](https://docs.notte.cc/api-reference/sessions/upload-session-file.md) ## Usage @@ -345,6 +342,8 @@ The SDK docs below are for generated-code editing and reference. They are not th - [DELETE Vault Delete](https://docs.notte.cc/api-reference/vaults/vault-delete.md) - [PATCH Vault Update](https://docs.notte.cc/api-reference/vaults/vault-update.md) + + ## SDK diff --git a/docs/src/scripts/generate_llms.py b/docs/src/scripts/generate_llms.py index 21be4b031..b59c55a5e 100644 --- a/docs/src/scripts/generate_llms.py +++ b/docs/src/scripts/generate_llms.py @@ -5,12 +5,19 @@ Each page is emitted as a bullet with title + description pulled from the page's YAML frontmatter. +The API section is rendered from the OpenAPI spec and kept between marker +comments. By default it is reused verbatim from the existing llms.txt so the +generator is deterministic and offline, which is what the pre-commit hook +needs; pass --refresh-openapi to fetch the live spec and rebuild it, which is +what the daily refresh workflow does. + Run from anywhere: - python3 src/scripts/generate_llms.py + python3 src/scripts/generate_llms.py [--refresh-openapi] """ from __future__ import annotations +import argparse import json import re import sys @@ -168,6 +175,41 @@ def render_openapi(spec: dict) -> list[str]: return lines +OPENAPI_END_MARKER = "" + + +def openapi_begin_marker(url: str) -> str: + return f"" + + +def cached_openapi_section(existing: str, url: str) -> list[str] | None: + """Return the marker-delimited API block for `url` from a previous llms.txt, or None.""" + begin = openapi_begin_marker(url) + start = existing.find(begin) + if start == -1: + return None + end = existing.find(OPENAPI_END_MARKER, start) + if end == -1: + return None + block = existing[start : end + len(OPENAPI_END_MARKER)] + return block.split("\n") + + +def openapi_section(url: str, *, refresh: bool, existing: str) -> list[str]: + """The API block for `url`: reused from `existing` unless refreshing or absent.""" + if not refresh: + cached = cached_openapi_section(existing, url) + if cached is not None: + return cached + [""] + print(f" warning: no cached openapi section for {url}, fetching live", file=sys.stderr) + try: + spec = fetch_openapi(url) + except Exception as e: + print(f" warning: failed to fetch openapi {url}: {e}", file=sys.stderr) + return [f"- OpenAPI spec: {url}", ""] + return [openapi_begin_marker(url), ""] + render_openapi(spec) + [OPENAPI_END_MARKER, ""] + + def read_frontmatter(page_path: str) -> dict: """Return parsed frontmatter dict for a nav page path.""" for ext in (".mdx", ".md"): @@ -226,7 +268,16 @@ def render_pages(pages: list, depth: int) -> list[str]: return lines -def main() -> int: +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--refresh-openapi", + action="store_true", + help="fetch the live OpenAPI spec and rebuild the API section instead of reusing the committed one", + ) + args = parser.parse_args(argv) + existing = OUTPUT.read_text(encoding="utf-8") if OUTPUT.exists() else "" + config = json.loads(DOCS_JSON.read_text()) site_name = config.get("name", "Docs") @@ -253,21 +304,11 @@ def main() -> int: for group in tab.get("groups", []): out += [f"## {group['group']}", ""] if "openapi" in group: - try: - spec = fetch_openapi(group["openapi"]) - out += render_openapi(spec) - except Exception as e: - print(f" warning: failed to fetch openapi {group['openapi']}: {e}", file=sys.stderr) - out += [f"- OpenAPI spec: {group['openapi']}", ""] + out += openapi_section(group["openapi"], refresh=args.refresh_openapi, existing=existing) out += render_pages(group.get("pages", []), depth=3) out += [""] if "openapi" in tab: - try: - spec = fetch_openapi(tab["openapi"]) - out += render_openapi(spec) - except Exception as e: - print(f" warning: failed to fetch openapi {tab['openapi']}: {e}", file=sys.stderr) - out += [f"- OpenAPI spec: {tab['openapi']}", ""] + out += openapi_section(tab["openapi"], refresh=args.refresh_openapi, existing=existing) OUTPUT.write_text("\n".join(out).rstrip() + "\n") print(f"wrote {OUTPUT.relative_to(SRC_DIR.parent)} ({OUTPUT.stat().st_size} bytes)") diff --git a/makefile b/makefile index 3870e0bb2..bca11417f 100644 --- a/makefile +++ b/makefile @@ -104,6 +104,12 @@ docs-sdk: docs-llms docs-llms: cd docs/src && uv run python scripts/generate_llms.py +# Rebuild the API section of llms.txt from the live OpenAPI spec. Only the +# refresh-llms workflow should need this; docs-llms reuses the committed section. +.PHONY: docs-llms-refresh +docs-llms-refresh: + cd docs/src && uv run python scripts/generate_llms.py --refresh-openapi + .PHONY: docs-agent-notice docs-agent-notice: diff --git a/tests/test_generate_llms_offline.py b/tests/test_generate_llms_offline.py new file mode 100644 index 000000000..7d71aa45e --- /dev/null +++ b/tests/test_generate_llms_offline.py @@ -0,0 +1,108 @@ +"""`generate_llms.py` reuses the committed API section unless asked to refresh it. + +The pre-commit hook regenerates llms.txt on every run, so the default path must +not depend on the live OpenAPI spec: any drift in the API between a local run and +CI would otherwise fail the hook on unrelated pull requests. +""" + +import importlib.util +from pathlib import Path +from types import ModuleType + +import pytest + +SCRIPT = Path(__file__).resolve().parent.parent / "docs" / "src" / "scripts" / "generate_llms.py" +URL = "https://api.example.test/openapi.json" + + +@pytest.fixture(scope="module") +def generate_llms() -> ModuleType: + spec = importlib.util.spec_from_file_location("generate_llms", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def existing_llms(generate_llms: ModuleType) -> str: + return "\n".join( + [ + "## APIs", + "", + generate_llms.openapi_begin_marker(URL), + "", + "## Sessions", + "", + "- [POST Session Start](https://docs.notte.cc/api-reference/sessions/session-start.md)", + "", + generate_llms.OPENAPI_END_MARKER, + "", + "## SDK", + ] + ) + + +def test_default_run_reuses_the_committed_section_without_fetching( + generate_llms: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + def no_network(url: str) -> dict: + raise AssertionError(f"fetched {url} in offline mode") + + monkeypatch.setattr(generate_llms, "fetch_openapi", no_network) + + section = generate_llms.openapi_section(URL, refresh=False, existing=existing_llms(generate_llms)) + + assert section[0] == generate_llms.openapi_begin_marker(URL) + assert "- [POST Session Start](https://docs.notte.cc/api-reference/sessions/session-start.md)" in section + assert section[-2] == generate_llms.OPENAPI_END_MARKER + + +def test_refresh_rebuilds_the_section_from_the_live_spec( + generate_llms: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + spec = { + "paths": { + "/sessions/start": { + "post": {"tags": ["sessions"], "summary": "Session Start", "operationId": "session_start"} + } + } + } + monkeypatch.setattr(generate_llms, "fetch_openapi", lambda url: spec) + + section = generate_llms.openapi_section(URL, refresh=True, existing="stale text without markers") + + assert section[0] == generate_llms.openapi_begin_marker(URL) + assert "- [POST Session Start](https://docs.notte.cc/api-reference/sessions/session-start.md)" in section + assert generate_llms.OPENAPI_END_MARKER in section + + +def test_missing_cached_section_falls_back_to_fetching( + generate_llms: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[str] = [] + + def fetch(url: str) -> dict: + calls.append(url) + return {"paths": {}} + + monkeypatch.setattr(generate_llms, "fetch_openapi", fetch) + + section = generate_llms.openapi_section(URL, refresh=False, existing="no markers here") + + assert calls == [URL] + assert section[0] == generate_llms.openapi_begin_marker(URL) + + +def test_failed_fetch_keeps_the_stub_line_for_the_refresh_workflow( + generate_llms: ModuleType, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def fetch(url: str) -> dict: + raise OSError("offline") + + monkeypatch.setattr(generate_llms, "fetch_openapi", fetch) + + section = generate_llms.openapi_section(URL, refresh=True, existing="") + + # refresh-llms.yml greps stderr for this phrase to refuse a degraded file + assert "failed to fetch openapi" in capsys.readouterr().err + assert section == [f"- OpenAPI spec: {URL}", ""] From d88d87e0a6befabdf8728a8c5abec413324711c4 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Sun, 6 Sep 2026 10:04:27 +0200 Subject: [PATCH 6/6] fix(fetch): preserve response bytes and honour the declared charset --- .../notte-core/src/notte_core/data/fetch.py | 41 +++++++++++++---- tests/sdk/test_fetch_helper.py | 44 ++++++++++++++++++- tests/test_fetch_helper.py | 16 +++++++ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/packages/notte-core/src/notte_core/data/fetch.py b/packages/notte-core/src/notte_core/data/fetch.py index 1b834c644..6356e955e 100644 --- a/packages/notte-core/src/notte_core/data/fetch.py +++ b/packages/notte-core/src/notte_core/data/fetch.py @@ -9,6 +9,7 @@ from __future__ import annotations +import base64 import io import json from collections.abc import Mapping @@ -97,20 +98,45 @@ def build_fetch_script( f"const init = {json.dumps(init)};" f"{abort}" f"const response = await fetch({json.dumps(request_url)}, init);" - "const text = await response.text();" + # ship the raw bytes as base64: `response.text()` would decode with + # replacement and lose any non-UTF-8 or binary body for good + "const bytes = new Uint8Array(await response.arrayBuffer());" + "let binary = '';" + "for (let i = 0; i < bytes.length; i += 0x8000) {" + " binary += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));" + "}" "const headers = {};" "response.headers.forEach((value, key) => { headers[key] = value; });" - "return JSON.stringify({status: response.status, url: response.url, headers: headers, text: text});" + "return JSON.stringify({status: response.status, url: response.url, headers: headers, body_b64: btoa(binary)});" "})()" ) +def _encoding_for(headers: Mapping[str, str], content: bytes) -> str | None: + """The charset the response declares, else utf-8 when the bytes are valid utf-8. + + Returning None leaves `requests` to detect the encoding from the bytes, which + is the right call for the odd legacy page that declares nothing. + """ + content_type = next((value for key, value in headers.items() if key.lower() == _CONTENT_TYPE), "") + for param in content_type.split(";")[1:]: + name, _, value = param.strip().partition("=") + if name.strip().lower() == "charset" and value: + return value.strip().strip("\"'") + try: + _ = content.decode("utf-8") + except UnicodeDecodeError: + return None + return "utf-8" + + def response_from_evaluated(raw: str) -> requests.Response: """Turn the envelope `build_fetch_script` returns into a `requests.Response`. A non-2xx status is a response, not an error; `raise_for_status()` raises - `requests.HTTPError` as usual. `url` is the final URL after redirects, and - the body is exposed through `text`, `content` and `json()`. + `requests.HTTPError` as usual. `url` is the final URL after redirects. + `content` holds the exact bytes the server sent, so binary bodies survive; + `text` decodes them with the declared charset, or utf-8 when none is given. """ try: payload: Any = json.loads(raw) @@ -123,7 +149,7 @@ def response_from_evaluated(raw: str) -> requests.Response: status_code = int(envelope["status"]) raw_headers: Any = envelope.get("headers") or {} headers = {str(key): str(value) for key, value in dict(raw_headers).items()} - text = str(envelope.get("text", "")) + content = base64.b64decode(str(envelope.get("body_b64", "")), validate=True) url = str(envelope.get("url", "")) except (KeyError, TypeError, ValueError) as exc: raise FetchResponseDecodeError(reason=str(exc)) from exc @@ -131,9 +157,8 @@ def response_from_evaluated(raw: str) -> requests.Response: response = requests.Response() response.status_code = status_code response.headers = CaseInsensitiveDict(headers) - # the browser already decoded the body; hand it back as utf-8 so `.text` round-trips - response.encoding = "utf-8" - response.raw = io.BytesIO(text.encode("utf-8")) + response.encoding = _encoding_for(headers, content) + response.raw = io.BytesIO(content) response.url = url try: response.reason = HTTPStatus(status_code).phrase diff --git a/tests/sdk/test_fetch_helper.py b/tests/sdk/test_fetch_helper.py index 9debae85a..25d27d6d6 100644 --- a/tests/sdk/test_fetch_helper.py +++ b/tests/sdk/test_fetch_helper.py @@ -1,5 +1,6 @@ """Remote `fetch()`: the request runs in the page via `evaluate_js` and comes back as a `requests.Response`.""" +import base64 import datetime as dt import json @@ -14,8 +15,22 @@ from tests.sdk.test_execute_raise_on_failure import over_the_wire, remote_session -def envelope(status: int = 200, text: str = '{"ok": true}', url: str = "https://example.com/api") -> str: - return json.dumps({"status": status, "url": url, "headers": {"content-type": "application/json"}, "text": text}) +def envelope( + status: int = 200, + text: str = '{"ok": true}', + url: str = "https://example.com/api", + content_type: str = "application/json", + body: bytes | None = None, +) -> str: + payload = body if body is not None else text.encode("utf-8") + return json.dumps( + { + "status": status, + "url": url, + "headers": {"content-type": content_type}, + "body_b64": base64.b64encode(payload).decode("ascii"), + } + ) def eval_result(markdown: str) -> ExecutionResult: @@ -136,6 +151,31 @@ def test_fetch_returns_http_errors_and_raises_only_when_asked() -> None: assert raised.value.response is response +def test_fetch_preserves_binary_bodies_byte_for_byte() -> None: + payload = b"caf\xe9\x00\xff" + session = remote_session( + over_the_wire(eval_result(envelope(content_type="application/octet-stream", body=payload))) + ) + + response = session.fetch("/blob") + + assert response.content == payload + # undeclared charset and not valid utf-8: left to requests' detection, never forced + assert response.encoding is None + + +def test_fetch_decodes_text_with_the_declared_charset() -> None: + payload = "caf\u00e9".encode("latin-1") + session = remote_session( + over_the_wire(eval_result(envelope(content_type="text/html; charset=ISO-8859-1", body=payload))) + ) + + response = session.fetch("/page") + + assert response.encoding == "ISO-8859-1" + assert response.text == "caf\u00e9" + + def test_fetch_rejects_an_unreadable_envelope() -> None: session = remote_session(over_the_wire(eval_result("not json"))) diff --git a/tests/test_fetch_helper.py b/tests/test_fetch_helper.py index a6c95b3af..8ec9000a3 100644 --- a/tests/test_fetch_helper.py +++ b/tests/test_fetch_helper.py @@ -29,6 +29,22 @@ async def test_afetch_json_reads_the_body() -> None: assert response.json() == {"a": 1} +@pytest.mark.asyncio +async def test_afetch_returns_binary_bodies_intact() -> None: + async with NotteSession(headless=True) as session: + _ = await session.aexecute(type="goto", url="https://www.example.com/") + + # a 1x1 PNG served from a data URL: bytes that are not valid utf-8 + response = await session.afetch( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + ) + + assert response.status_code == 200 + assert response.headers["Content-Type"] == "image/png" + assert response.content.startswith(b"\x89PNG\r\n\x1a\n") + assert len(response.content) == 70 + + @pytest.mark.asyncio async def test_afetch_network_failure_raises_the_js_error() -> None: async with NotteSession(headless=True) as session: