Skip to content

Commit 7d04de7

Browse files
lidellidel
committed
feat(gateway): add content size limits for responses
Add two new Config options for gateway operators to limit responses based on content size read from the UnixFS root block: - MaxDeserializedResponseSize: caps deserialized responses only, trustless formats (raw, CAR) are not affected - MaxUnixFSDAGResponseSize: caps all response formats including raw blocks, CAR, and TAR Both return 501 Not Implemented with a message directing users to run their own IPFS node for large content. - gateway.go: add config fields with documentation - handler.go: add exceedsMax* helper methods - handler_defaults.go: check both limits using bytesSize - handler_block.go: check DAG limit using existing block size - handler_codec.go: check DAG limit using existing block size - handler_car.go: conditional Head call only when limit is set - handler_tar.go: check DAG limit using existing file.Size() - gateway_test.go: tests for both limits across all formats - CHANGELOG.md: document new config options
1 parent a696a54 commit 7d04de7

File tree

9 files changed

+385
-0
lines changed

9 files changed

+385
-0
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ The following emojis are used to highlight certain changes:
1616

1717
### Added
1818

19+
- `gateway`: `Config.MaxDeserializedResponseSize` allows setting a maximum file/directory size for deserialized gateway responses. Content exceeding this limit returns `501 Not Implemented`, directing users to run their own IPFS node. Trustless response formats (`application/vnd.ipld.raw`, `application/vnd.ipld.car`) are not affected. The size is read from the UnixFS root block, so no extra block fetches are needed for the check. [#1129](https://github.com/ipfs/boxo/pull/1129)
20+
- `gateway`: `Config.MaxUnixFSDAGResponseSize` allows setting a maximum content size applied to all response formats (deserialized, raw blocks, CAR, TAR). Content exceeding this limit returns `501 Not Implemented`. For most handlers the check reuses size information already available in the request path; for CAR responses a lightweight `Head` call is made only when the limit is configured. [#1129](https://github.com/ipfs/boxo/pull/1129)
21+
1922
### Changed
2023

2124
### Removed

‎gateway/gateway.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,31 @@ type Config struct {
150150
// (e.g., Cloudflare's 5GB limit). A value of 0 disables this limit.
151151
MaxRangeRequestFileSize int64
152152

153+
// MaxDeserializedResponseSize is the maximum file or directory DAG size
154+
// in bytes for deserialized responses. When set to a value greater than 0,
155+
// requests for UnixFS content larger than this limit will return
156+
// 501 Not Implemented, directing users to run their own IPFS node for
157+
// large content. This applies to both regular and range requests: if the
158+
// underlying file exceeds the limit, even a small range is rejected.
159+
// No additional block fetches are needed; size is already available from
160+
// the request's normal processing of the UnixFS root block.
161+
// A value of 0 disables this limit. Only affects deserialized responses;
162+
// trustless formats (application/vnd.ipld.raw, application/vnd.ipld.car)
163+
// are not affected.
164+
MaxDeserializedResponseSize int64
165+
166+
// MaxUnixFSDAGResponseSize is the maximum UnixFS file or directory DAG
167+
// size in bytes, applied to all response formats: deserialized, raw
168+
// blocks, CAR, and TAR. When set to a value greater than 0, any request
169+
// whose resolved content exceeds this limit will return
170+
// 501 Not Implemented, regardless of response format. This allows
171+
// gateway operators to cap bandwidth across all response types.
172+
// Most handlers reuse the size already available from normal request
173+
// processing; the CAR handler performs a lightweight Head call (root
174+
// block is then cached for the subsequent CAR traversal).
175+
// A value of 0 disables this limit.
176+
MaxUnixFSDAGResponseSize int64
177+
153178
// MaxRequestDuration is the maximum total time a request can take.
154179
// Unlike RetrievalTimeout (which resets on each data write and catches
155180
// stalled transfers), this is an absolute deadline for the entire request.

‎gateway/gateway_test.go‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1640,6 +1640,290 @@ func TestMaxRangeRequestFileSize(t *testing.T) {
16401640
})
16411641
}
16421642

1643+
func TestMaxDeserializedResponseSize(t *testing.T) {
1644+
backend, root := newMockBackend(t, "fixtures.car")
1645+
1646+
// "fnord" file is 5 bytes, lives at subdir/fnord
1647+
p, err := path.Join(path.FromCid(root), "subdir", "fnord")
1648+
require.NoError(t, err)
1649+
1650+
ctx := t.Context()
1651+
1652+
k, err := backend.resolvePathNoRootsReturned(ctx, p)
1653+
require.NoError(t, err)
1654+
1655+
t.Run("GET exceeding limit returns 501", func(t *testing.T) {
1656+
ts := newTestServerWithConfig(t, backend, Config{
1657+
DeserializedResponses: true,
1658+
MaxDeserializedResponseSize: 4, // smaller than "fnord" (5 bytes)
1659+
})
1660+
1661+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1662+
require.NoError(t, err)
1663+
1664+
res := mustDoWithoutRedirect(t, req)
1665+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1666+
1667+
body, err := io.ReadAll(res.Body)
1668+
require.NoError(t, err)
1669+
require.Contains(t, string(body), "not supported for content larger than 4 bytes")
1670+
require.Contains(t, string(body), "https://docs.ipfs.tech/install/")
1671+
})
1672+
1673+
t.Run("range request for file exceeding limit returns 501", func(t *testing.T) {
1674+
ts := newTestServerWithConfig(t, backend, Config{
1675+
DeserializedResponses: true,
1676+
MaxDeserializedResponseSize: 4, // smaller than "fnord" (5 bytes)
1677+
})
1678+
1679+
// Even though range is only 2 bytes, the file itself is 5 bytes
1680+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1681+
require.NoError(t, err)
1682+
req.Header.Set("Range", "bytes=0-1")
1683+
1684+
res := mustDoWithoutRedirect(t, req)
1685+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1686+
1687+
body, err := io.ReadAll(res.Body)
1688+
require.NoError(t, err)
1689+
require.Contains(t, string(body), "not supported for content larger than 4 bytes")
1690+
})
1691+
1692+
t.Run("HEAD exceeding limit returns 501", func(t *testing.T) {
1693+
ts := newTestServerWithConfig(t, backend, Config{
1694+
DeserializedResponses: true,
1695+
MaxDeserializedResponseSize: 4,
1696+
})
1697+
1698+
req, err := http.NewRequest(http.MethodHead, ts.URL+k.String(), nil)
1699+
require.NoError(t, err)
1700+
1701+
res := mustDoWithoutRedirect(t, req)
1702+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1703+
})
1704+
1705+
t.Run("GET within limit works", func(t *testing.T) {
1706+
ts := newTestServerWithConfig(t, backend, Config{
1707+
DeserializedResponses: true,
1708+
MaxDeserializedResponseSize: 1000,
1709+
})
1710+
1711+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1712+
require.NoError(t, err)
1713+
1714+
res := mustDoWithoutRedirect(t, req)
1715+
require.Equal(t, http.StatusOK, res.StatusCode)
1716+
1717+
body, err := io.ReadAll(res.Body)
1718+
require.NoError(t, err)
1719+
require.Equal(t, "fnord", string(body))
1720+
})
1721+
1722+
t.Run("disabled when set to 0", func(t *testing.T) {
1723+
ts := newTestServerWithConfig(t, backend, Config{
1724+
DeserializedResponses: true,
1725+
MaxDeserializedResponseSize: 0,
1726+
})
1727+
1728+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1729+
require.NoError(t, err)
1730+
1731+
res := mustDoWithoutRedirect(t, req)
1732+
require.Equal(t, http.StatusOK, res.StatusCode)
1733+
1734+
body, err := io.ReadAll(res.Body)
1735+
require.NoError(t, err)
1736+
require.Equal(t, "fnord", string(body))
1737+
})
1738+
1739+
t.Run("raw format query param bypasses limit", func(t *testing.T) {
1740+
ts := newTestServerWithConfig(t, backend, Config{
1741+
DeserializedResponses: true,
1742+
MaxDeserializedResponseSize: 1, // 1 byte, way below any content
1743+
})
1744+
1745+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String()+"?format=raw", nil)
1746+
require.NoError(t, err)
1747+
1748+
res := mustDoWithoutRedirect(t, req)
1749+
require.Equal(t, http.StatusOK, res.StatusCode)
1750+
})
1751+
1752+
t.Run("raw Accept header bypasses limit", func(t *testing.T) {
1753+
ts := newTestServerWithConfig(t, backend, Config{
1754+
DeserializedResponses: true,
1755+
MaxDeserializedResponseSize: 1,
1756+
})
1757+
1758+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1759+
require.NoError(t, err)
1760+
req.Header.Set("Accept", "application/vnd.ipld.raw")
1761+
1762+
res := mustDoWithoutRedirect(t, req)
1763+
require.Equal(t, http.StatusOK, res.StatusCode)
1764+
})
1765+
1766+
t.Run("car format query param bypasses limit", func(t *testing.T) {
1767+
ts := newTestServerWithConfig(t, backend, Config{
1768+
DeserializedResponses: true,
1769+
MaxDeserializedResponseSize: 1,
1770+
})
1771+
1772+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String()+"?format=car", nil)
1773+
require.NoError(t, err)
1774+
1775+
res := mustDoWithoutRedirect(t, req)
1776+
require.Equal(t, http.StatusOK, res.StatusCode)
1777+
})
1778+
1779+
t.Run("car Accept header bypasses limit", func(t *testing.T) {
1780+
ts := newTestServerWithConfig(t, backend, Config{
1781+
DeserializedResponses: true,
1782+
MaxDeserializedResponseSize: 1,
1783+
})
1784+
1785+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1786+
require.NoError(t, err)
1787+
req.Header.Set("Accept", "application/vnd.ipld.car")
1788+
1789+
res := mustDoWithoutRedirect(t, req)
1790+
require.Equal(t, http.StatusOK, res.StatusCode)
1791+
})
1792+
}
1793+
1794+
func TestMaxUnixFSDAGResponseSize(t *testing.T) {
1795+
backend, root := newMockBackend(t, "fixtures.car")
1796+
1797+
// "fnord" file is 5 bytes, lives at subdir/fnord
1798+
p, err := path.Join(path.FromCid(root), "subdir", "fnord")
1799+
require.NoError(t, err)
1800+
1801+
ctx := t.Context()
1802+
1803+
k, err := backend.resolvePathNoRootsReturned(ctx, p)
1804+
require.NoError(t, err)
1805+
1806+
t.Run("deserialized GET exceeding limit returns 501", func(t *testing.T) {
1807+
ts := newTestServerWithConfig(t, backend, Config{
1808+
DeserializedResponses: true,
1809+
MaxUnixFSDAGResponseSize: 4,
1810+
})
1811+
1812+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1813+
require.NoError(t, err)
1814+
1815+
res := mustDoWithoutRedirect(t, req)
1816+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1817+
1818+
body, err := io.ReadAll(res.Body)
1819+
require.NoError(t, err)
1820+
require.Contains(t, string(body), "not supported for content larger than 4 bytes")
1821+
require.Contains(t, string(body), "https://docs.ipfs.tech/install/")
1822+
})
1823+
1824+
t.Run("deserialized range request for file exceeding limit returns 501", func(t *testing.T) {
1825+
ts := newTestServerWithConfig(t, backend, Config{
1826+
DeserializedResponses: true,
1827+
MaxUnixFSDAGResponseSize: 4,
1828+
})
1829+
1830+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1831+
require.NoError(t, err)
1832+
req.Header.Set("Range", "bytes=0-1")
1833+
1834+
res := mustDoWithoutRedirect(t, req)
1835+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1836+
})
1837+
1838+
t.Run("raw format exceeding limit returns 501", func(t *testing.T) {
1839+
ts := newTestServerWithConfig(t, backend, Config{
1840+
DeserializedResponses: true,
1841+
MaxUnixFSDAGResponseSize: 4,
1842+
})
1843+
1844+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String()+"?format=raw", nil)
1845+
require.NoError(t, err)
1846+
1847+
res := mustDoWithoutRedirect(t, req)
1848+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1849+
})
1850+
1851+
t.Run("raw Accept header exceeding limit returns 501", func(t *testing.T) {
1852+
ts := newTestServerWithConfig(t, backend, Config{
1853+
DeserializedResponses: true,
1854+
MaxUnixFSDAGResponseSize: 4,
1855+
})
1856+
1857+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1858+
require.NoError(t, err)
1859+
req.Header.Set("Accept", "application/vnd.ipld.raw")
1860+
1861+
res := mustDoWithoutRedirect(t, req)
1862+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1863+
})
1864+
1865+
t.Run("car format exceeding limit returns 501", func(t *testing.T) {
1866+
ts := newTestServerWithConfig(t, backend, Config{
1867+
DeserializedResponses: true,
1868+
MaxUnixFSDAGResponseSize: 4,
1869+
})
1870+
1871+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String()+"?format=car", nil)
1872+
require.NoError(t, err)
1873+
1874+
res := mustDoWithoutRedirect(t, req)
1875+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1876+
})
1877+
1878+
t.Run("car Accept header exceeding limit returns 501", func(t *testing.T) {
1879+
ts := newTestServerWithConfig(t, backend, Config{
1880+
DeserializedResponses: true,
1881+
MaxUnixFSDAGResponseSize: 4,
1882+
})
1883+
1884+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1885+
require.NoError(t, err)
1886+
req.Header.Set("Accept", "application/vnd.ipld.car")
1887+
1888+
res := mustDoWithoutRedirect(t, req)
1889+
require.Equal(t, http.StatusNotImplemented, res.StatusCode)
1890+
})
1891+
1892+
t.Run("GET within limit works", func(t *testing.T) {
1893+
ts := newTestServerWithConfig(t, backend, Config{
1894+
DeserializedResponses: true,
1895+
MaxUnixFSDAGResponseSize: 1000,
1896+
})
1897+
1898+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1899+
require.NoError(t, err)
1900+
1901+
res := mustDoWithoutRedirect(t, req)
1902+
require.Equal(t, http.StatusOK, res.StatusCode)
1903+
1904+
body, err := io.ReadAll(res.Body)
1905+
require.NoError(t, err)
1906+
require.Equal(t, "fnord", string(body))
1907+
})
1908+
1909+
t.Run("disabled when set to 0", func(t *testing.T) {
1910+
ts := newTestServerWithConfig(t, backend, Config{
1911+
DeserializedResponses: true,
1912+
MaxUnixFSDAGResponseSize: 0,
1913+
})
1914+
1915+
req, err := http.NewRequest(http.MethodGet, ts.URL+k.String(), nil)
1916+
require.NoError(t, err)
1917+
1918+
res := mustDoWithoutRedirect(t, req)
1919+
require.Equal(t, http.StatusOK, res.StatusCode)
1920+
1921+
body, err := io.ReadAll(res.Body)
1922+
require.NoError(t, err)
1923+
require.Equal(t, "fnord", string(body))
1924+
})
1925+
}
1926+
16431927
func TestValidateConfig_MaxRequestDuration(t *testing.T) {
16441928
t.Parallel()
16451929

‎gateway/handler.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1125,3 +1125,27 @@ func (i *handler) getTemplateGlobalData(r *http.Request, contentPath path.Path)
11251125
func (i *handler) webError(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
11261126
webError(w, r, i.config, err, defaultCode)
11271127
}
1128+
1129+
// exceedsMaxUnixFSDAGResponseSize checks whether sz exceeds the configured
1130+
// MaxUnixFSDAGResponseSize. If it does, it writes a 501 error and returns true.
1131+
// Returns false (no-op) when the limit is disabled or not exceeded.
1132+
func (i *handler) exceedsMaxUnixFSDAGResponseSize(w http.ResponseWriter, r *http.Request, sz int64) bool {
1133+
if i.config.MaxUnixFSDAGResponseSize > 0 && sz > i.config.MaxUnixFSDAGResponseSize {
1134+
err := fmt.Errorf("responses are not supported for content larger than %d bytes: for large content, run your own IPFS node (https://docs.ipfs.tech/install/)", i.config.MaxUnixFSDAGResponseSize)
1135+
i.webError(w, r, err, http.StatusNotImplemented)
1136+
return true
1137+
}
1138+
return false
1139+
}
1140+
1141+
// exceedsMaxDeserializedResponseSize checks whether sz exceeds the configured
1142+
// MaxDeserializedResponseSize. If it does, it writes a 501 error and returns true.
1143+
// Returns false (no-op) when the limit is disabled or not exceeded.
1144+
func (i *handler) exceedsMaxDeserializedResponseSize(w http.ResponseWriter, r *http.Request, sz int64) bool {
1145+
if i.config.MaxDeserializedResponseSize > 0 && sz > i.config.MaxDeserializedResponseSize {
1146+
err := fmt.Errorf("deserialized responses are not supported for content larger than %d bytes: for large content, run your own IPFS node (https://docs.ipfs.tech/install/)", i.config.MaxDeserializedResponseSize)
1147+
i.webError(w, r, err, http.StatusNotImplemented)
1148+
return true
1149+
}
1150+
return false
1151+
}

‎gateway/handler_block.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h
4444
return false
4545
}
4646

47+
if i.exceedsMaxUnixFSDAGResponseSize(w, r, sz) {
48+
return false
49+
}
50+
4751
if !i.seekToStartOfFirstRange(w, r, data, sz) {
4852
return false
4953
}

‎gateway/handler_car.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,21 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
7373
return false
7474
}
7575

76+
// Check DAG size limit before streaming the CAR. This requires a
77+
// lightweight Head call; the root block is cached in the blockstore
78+
// so GetCAR will not re-fetch it from the network.
79+
if i.config.MaxUnixFSDAGResponseSize > 0 {
80+
_, headResp, headErr := i.backend.Head(ctx, rq.immutablePath)
81+
if headErr == nil {
82+
sz := headResp.bytesSize
83+
headResp.Close()
84+
if i.exceedsMaxUnixFSDAGResponseSize(w, r, sz) {
85+
return false
86+
}
87+
}
88+
// If Head fails, let GetCAR surface the error with proper handling.
89+
}
90+
7691
md, carFile, err := i.backend.GetCAR(ctx, rq.immutablePath, params)
7792
if !i.handleRequestErrors(w, r, rq.contentPath, err) {
7893
return false

0 commit comments

Comments
 (0)