From a3408c6d6798176b222f092d2d166e2eeb13df98 Mon Sep 17 00:00:00 2001
From: pvbang <mekongai.official@gmail.com>
Date: Sat, 13 Jun 2026 06:03:54 +0000
Subject: [PATCH] feat: add noLinks and lang parameters to web.scrape method

Add support for the missing noLinks and lang parameters in the web
scrape function to maintain feature parity with the REST API.

- noLinks (bool): Whether to exclude Markdown links from the response
- lang (str): Language to request content in (ISO 639-1 code)

Both parameters are optional and backward-compatible.

Closes #19
---
 supadata/web.py      | 17 +++++++--
 tests/test_client.py | 84 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/supadata/web.py b/supadata/web.py
index e69c9e0..ef72411 100644
--- a/supadata/web.py
+++ b/supadata/web.py
@@ -16,11 +16,18 @@ def __init__(self, request_handler):
         """
         self._request = request_handler
 
-    def scrape(self, url: str) -> Scrape:
+    def scrape(
+        self,
+        url: str,
+        no_links: Optional[bool] = None,
+        lang: Optional[str] = None,
+    ) -> Scrape:
         """Scrape content from a web page.
 
         Args:
             url: URL to scrape
+            no_links: Whether to exclude Markdown links from the response
+            lang: Language to request content in (ISO 639-1 code, e.g. "en", "es")
 
         Returns:
             Scrape object containing the extracted content
@@ -28,7 +35,13 @@ def scrape(self, url: str) -> Scrape:
         Raises:
             SupadataError: If the API request fails
         """
-        response = self._request("GET", "/web/scrape", params={"url": url})
+        params: Dict[str, Union[str, bool]] = {"url": url}
+        if no_links is not None:
+            params["noLinks"] = str(no_links).lower()
+        if lang is not None:
+            params["lang"] = lang
+
+        response = self._request("GET", "/web/scrape", params=params)
         return Scrape(**response)
 
     def map(self, url: str) -> Map:
diff --git a/tests/test_client.py b/tests/test_client.py
index f52136b..6f6a2b3 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -130,6 +130,90 @@ def test_scrape(client: Supadata, requests_mock) -> None:
     assert content.count_characters == 100
 
 
+def test_scrape_with_no_links(client: Supadata, requests_mock) -> None:
+    """Test web scraping with noLinks parameter."""
+    url = "https://test.com"
+    mock_response = {
+        "url": url,
+        "content": "Test content without links",
+        "name": "Test Page",
+        "description": "A test page",
+        "ogUrl": None,
+        "countCharacters": 50,
+        "urls": [],
+    }
+    requests_mock.get(f"{client.base_url}/web/scrape", json=mock_response)
+
+    content = client.web.scrape(url=url, no_links=True)
+    assert isinstance(content, Scrape)
+    assert content.content == "Test content without links"
+    # Verify the noLinks parameter was sent in the request
+    assert "noLinks=true" in requests_mock.last_request.url
+
+
+def test_scrape_with_lang(client: Supadata, requests_mock) -> None:
+    """Test web scraping with lang parameter."""
+    url = "https://test.com"
+    mock_response = {
+        "url": url,
+        "content": "Contenido de prueba en español",
+        "name": "Página de prueba",
+        "description": "Una página de prueba",
+        "ogUrl": None,
+        "countCharacters": 80,
+        "urls": [],
+    }
+    requests_mock.get(f"{client.base_url}/web/scrape", json=mock_response)
+
+    content = client.web.scrape(url=url, lang="es")
+    assert isinstance(content, Scrape)
+    assert content.content == "Contenido de prueba en español"
+    # Verify the lang parameter was sent in the request
+    assert "lang=es" in requests_mock.last_request.url
+
+
+def test_scrape_with_no_links_and_lang(client: Supadata, requests_mock) -> None:
+    """Test web scraping with both noLinks and lang parameters."""
+    url = "https://test.com"
+    mock_response = {
+        "url": url,
+        "content": "Contenu sans liens",
+        "name": "Page de test",
+        "description": "Une page de test",
+        "ogUrl": None,
+        "countCharacters": 40,
+        "urls": [],
+    }
+    requests_mock.get(f"{client.base_url}/web/scrape", json=mock_response)
+
+    content = client.web.scrape(url=url, no_links=True, lang="fr")
+    assert isinstance(content, Scrape)
+    assert content.content == "Contenu sans liens"
+    # Verify both parameters were sent in the request
+    assert "noLinks=true" in requests_mock.last_request.url
+    assert "lang=fr" in requests_mock.last_request.url
+
+
+def test_scrape_with_no_links_false(client: Supadata, requests_mock) -> None:
+    """Test web scraping with noLinks=False parameter."""
+    url = "https://test.com"
+    mock_response = {
+        "url": url,
+        "content": "[Link](https://example.com) content with links",
+        "name": "Test Page",
+        "description": "A test page",
+        "ogUrl": None,
+        "countCharacters": 60,
+        "urls": ["https://example.com"],
+    }
+    requests_mock.get(f"{client.base_url}/web/scrape", json=mock_response)
+
+    content = client.web.scrape(url=url, no_links=False)
+    assert isinstance(content, Scrape)
+    # Verify the noLinks parameter was sent as "false"
+    assert "noLinks=false" in requests_mock.last_request.url
+
+
 def test_map(client: Supadata, requests_mock) -> None:
     """Test site mapping."""
     url = "https://test.com"