From 15745f93e0586663fb9e1103ef98be35b5e59409 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 06:31:56 +0000 Subject: [PATCH 01/30] test/nginx/request(): use WHATWG Responses Simplifies test `request()` implementation to use standard `Response` classes. The `request()` function aims to be as similar to `fetch()` as possible, while allowing for non-standard behaviour necessary for tests. Not using `Response` was an oversight in the initial implementation. --- test/nginx/src/mocha/request.js | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/test/nginx/src/mocha/request.js b/test/nginx/src/mocha/request.js index 509a8eff3..da76bcaf3 100644 --- a/test/nginx/src/mocha/request.js +++ b/test/nginx/src/mocha/request.js @@ -1,5 +1,4 @@ const { isIPv6 } = require('node:net'); -const { Readable } = require('node:stream'); module.exports = request; @@ -16,29 +15,11 @@ function request(url, { body, ...options }={}) { const req = getProtocolImplFrom(url).request({ ...options, ...preserve(url) }, res => { res.on('error', reject); - const body = new Readable({ read:() => {} }); - res.on('error', err => body.destroy(err)); - res.on('data', data => body.push(data)); - res.on('end', () => body.push(null)); - - const text = () => new Promise((resolve, reject) => { - const chunks = []; - body.on('error', reject); - body.on('data', data => chunks.push(data)); - body.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); - }); - - const status = res.statusCode; - - resolve({ - status, - ok: status >= 200 && status < 300, - statusText: res.statusText, - body, - text, - json: async () => JSON.parse(await text()), + resolve(new Response(res, { + status: res.statusCode, + statusText: res.statusMessage, headers: new Headers(res.headers), - }); + })); }); req.on('error', reject); if(body !== undefined) req.write(body); From c03fbd37474628e642826e8a7baa63a33cbd3e08 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 06:56:06 +0000 Subject: [PATCH 02/30] re-import tests --- test/nginx/mock-http-server/index.js | 29 ++++++++++++++++++++ test/nginx/src/lib.js | 5 ++++ test/nginx/src/mocha/nginx.spec.js | 41 ++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index e060af017..b6039405d 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -1,9 +1,12 @@ +const { Readable } = require('node:stream'); + const express = require('express'); const port = process.env.PORT || 80; const log = (...args) => console.log('[mock-http-server]', ...args); const requests = []; +let openProcessorCount = 0; const app = express(); app.set('case sensitive routing', true); @@ -29,9 +32,35 @@ app.get('/health', (req, res) => res.send('OK')); app.get('/request-log', (req, res) => res.json(requests)); app.get('/reset', (req, res) => { requests.length = 0; + openProcessorCount = 0; res.json('OK'); }); +app.get('/v1/endless.csv', (req, res) => { + // TODO confirm from IRL what headers should be set, e.g. + // content-type + // content-length + // transfer-encoding + + ++openProcessorCount; + let rowCount = 0; + const randomStream = new Readable({ + read() { + randomStream.pipe(`${++rowCount},${new Date().toISOString()},${Math.random()}`, 'utf-8'); + }, + }); + randomStream.pipe(`row_number,timestamp,random-number`, 'utf-8'); + + randomStream.pipe(res); + req.on('close', () => { + randomStream.destroy(); + openProcessorCount = 0; + }); +}); +app.get('open-processor-count', (req, res) => { + res.send(openProcessorCount); +}); + app.get('/v1/reflect-headers', (req, res) => res.json(req.headers)); // Central-Backend can set Cache headers and those should have highest precedence diff --git a/test/nginx/src/lib.js b/test/nginx/src/lib.js index efc8af20c..b7484bbc6 100644 --- a/test/nginx/src/lib.js +++ b/test/nginx/src/lib.js @@ -10,6 +10,7 @@ module.exports = { assertSentryReceived, requestSentryMock, resetSentryMock, + sleep, }; async function assertSentryReceived(...expectedRequests) { @@ -56,3 +57,7 @@ function requestSentryMock(opts) { req.end(); }); } + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 32e7b7067..2b8b51e6b 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -5,6 +5,7 @@ const { assertSentryReceived, requestSentryMock, resetSentryMock, + sleep, } = require('../lib'); const request = require('./request'); @@ -436,6 +437,46 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); }); + describe('response buffering', () => { + async function assertOpenResponseProcessors(expectedCount) { + const res = await apiFetch('/open-processor-count'); + assert.isTrue(res.ok); + + const body = await res.text(); + const actualCount = Number(body); + + assert.equal(actualCount, expectedCount); + } + + it('should buffer responses in nginx, not backend services', async () => { + let controller; + + try { + // given + controller = new AbortController(); + const { signal } = controller; + + // when + const res = await apiFetch('/v1/endless.csv', { throttleAt:KBperSecond(100), signal }); + const reader = res.body.getReader(); + const { done, value } = reader.read(); + console.log({ done, value }); + + // then + await sleep(100); // give a chance for something to download // FIXME confirm this is required + assert.equal(res.status, 200); + // and + await assertOpenResponseProcessors(1); + } finally { + controller.abort(); + } + }); + + function KBperSecond(kilobytes) { + return 1000 * kilobytes; + } + }); + it('should serve generated client-config.json', async () => { // when const res = await apiFetch('/client-config.json'); From ab17292e39014a8c542562e2925d12a5df3d0ce2 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 06:57:59 +0000 Subject: [PATCH 03/30] just run the relevant test, once --- test/nginx/src/mocha/nginx.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 2b8b51e6b..e675a1137 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -290,7 +290,7 @@ describe('nginx config', () => { resetBackendMock(), ])); - describe('SSL_TYPE=selfsign', () => { + describe.only('SSL_TYPE=selfsign', () => { const { fetchHttp, fetchHttp6, fetchHttps, fetchHttps6 } = fetchFunctionsForPorts(9000, 9001); it('HTTP should forward to HTTPS', async () => { @@ -363,7 +363,7 @@ describe('nginx config', () => { }); }); - describe('SSL_TYPE=upstream', () => { + describe.skip('SSL_TYPE=upstream', () => { const { fetchHttp, fetchHttp6, fetchHttps, fetchHttps6 } = fetchFunctionsForPorts(10000, 10001); it('should not respond to HTTPS requests (IPv4)', async () => { @@ -437,7 +437,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); }); - describe('response buffering', () => { + describe.only('response buffering', () => { async function assertOpenResponseProcessors(expectedCount) { const res = await apiFetch('/open-processor-count'); assert.isTrue(res.ok); From 63934aa1c95e22d40661b11e588e48090c21aca7 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 07:20:51 +0000 Subject: [PATCH 04/30] make teh strewam finish --- test/nginx/mock-http-server/index.js | 35 ++++++++++++++++++++-------- test/nginx/src/mocha/nginx.spec.js | 13 +++++++---- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index b6039405d..f03a1ff64 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -36,28 +36,43 @@ app.get('/reset', (req, res) => { res.json('OK'); }); -app.get('/v1/endless.csv', (req, res) => { +app.get('/v1/25MB.csv', (req, res) => { // TODO confirm from IRL what headers should be set, e.g. // content-type // content-length // transfer-encoding ++openProcessorCount; - let rowCount = 0; - const randomStream = new Readable({ - read() { - randomStream.pipe(`${++rowCount},${new Date().toISOString()},${Math.random()}`, 'utf-8'); - }, - }); - randomStream.pipe(`row_number,timestamp,random-number`, 'utf-8'); + async function* generateCsv(targetByteLength) { + let rowCount = 0; + let written = 0; + + const writeLine = line => { + const buf = Buffer.from(line + '\n', 'utf8'); + written += buf.byteLength; + console.log('Writing; total written:', written); + return buf; + }; + + yield writeLine(`row_number,timestamp,random-number`); + + while(written < targetByteLength) { + yield writeLine(`${++rowCount},${new Date().toISOString()},${Math.random()}`); + } + } + + const randomStream = Readable.from(generateCsv(25_000_000)); randomStream.pipe(res); req.on('close', () => { randomStream.destroy(); - openProcessorCount = 0; + --openProcessorCount; }); }); -app.get('open-processor-count', (req, res) => { +app.get('/open-processor-count', (req, res) => { + console.log(` + /open-processor-count called; count: ${openProcessorCount} + `); res.send(openProcessorCount); }); diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index e675a1137..52f0adf60 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -439,10 +439,11 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward describe.only('response buffering', () => { async function assertOpenResponseProcessors(expectedCount) { - const res = await apiFetch('/open-processor-count'); + const res = await request(`http://localhost:8383/open-processor-count`); assert.isTrue(res.ok); const body = await res.text(); + console.log(`[open-processor-count: ${body}]`); const actualCount = Number(body); assert.equal(actualCount, expectedCount); @@ -457,16 +458,20 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward const { signal } = controller; // when - const res = await apiFetch('/v1/endless.csv', { throttleAt:KBperSecond(100), signal }); + const res = await apiFetch('/v1/25MB.csv', { throttleAt:KBperSecond(100), signal }); const reader = res.body.getReader(); - const { done, value } = reader.read(); + const { done, value } = await reader.read(); console.log({ done, value }); // then - await sleep(100); // give a chance for something to download // FIXME confirm this is required assert.equal(res.status, 200); // and await assertOpenResponseProcessors(1); + + // when + await sleep(100); // should be long enough for nginx to download the full stream from service + // then + await assertOpenResponseProcessors(0); } finally { controller.abort(); } From 07dd10db78443ba49116eb62f76b4825972e8808 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 08:07:05 +0000 Subject: [PATCH 05/30] wip --- files/nginx/odk.conf.template | 10 +++++++++- test/nginx/mock-http-server/index.js | 24 ++++++++++++++++-------- test/nginx/src/mocha/nginx.spec.js | 12 +++++------- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 4a5551c7b..05d5c906b 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -200,6 +200,8 @@ server { # central-backend location ~ ^/v\d { + error_log /dev/stderr info; + proxy_hide_header Content-Security-Policy; add_header Content-Security-Policy $central_backend_csp always; @@ -211,8 +213,14 @@ server { # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; - proxy_buffering off; + proxy_buffering on; proxy_read_timeout 2m; + proxy_temp_path /tmp; + proxy_max_temp_file_size 64m; + proxy_temp_file_write_size 64m; + proxy_buffers 64 64k; + proxy_buffer_size 64k; + proxy_busy_buffers_size 128k; } location @blank.html { diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index f03a1ff64..83823d42e 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -48,23 +48,31 @@ app.get('/v1/25MB.csv', (req, res) => { let rowCount = 0; let written = 0; - const writeLine = line => { - const buf = Buffer.from(line + '\n', 'utf8'); - written += buf.byteLength; - console.log('Writing; total written:', written); - return buf; - }; + const batchSize = Math.pow(2, 16); - yield writeLine(`row_number,timestamp,random-number`); + const header = Buffer.from('row_number,timestamp,random-number\n', 'utf8'); + written += header.byteLength; + yield header; while(written < targetByteLength) { - yield writeLine(`${++rowCount},${new Date().toISOString()},${Math.random()}`); + await new Promise(resolve => setTimeout(resolve, 1)); + const batch = Buffer.allocUnsafe(batchSize); + let bufpos = 0; + while(bufpos < batchSize) { + const line = `${++rowCount},${new Date().toISOString()},${Math.random()}\n`; + bufpos += batch.write(line, bufpos, 'utf8'); + } + written += batchSize; + console.log('written:', written); + yield batch; } } + // TODO up this to 100MiB const randomStream = Readable.from(generateCsv(25_000_000)); randomStream.pipe(res); req.on('close', () => { + console.log('req.closed; destroying randomStream'); randomStream.destroy(); --openProcessorCount; }); diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 52f0adf60..cc0e94139 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -449,7 +449,9 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward assert.equal(actualCount, expectedCount); } - it('should buffer responses in nginx, not backend services', async () => { + it('should buffer responses in nginx, not backend services', async function() { + this.timeout(20_000); // FIXME hopefully remove this + let controller; try { @@ -458,7 +460,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward const { signal } = controller; // when - const res = await apiFetch('/v1/25MB.csv', { throttleAt:KBperSecond(100), signal }); + const res = await apiFetch('/v1/25MB.csv', { signal }); const reader = res.body.getReader(); const { done, value } = await reader.read(); console.log({ done, value }); @@ -469,17 +471,13 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward await assertOpenResponseProcessors(1); // when - await sleep(100); // should be long enough for nginx to download the full stream from service + await sleep(10_000); // should be long enough for nginx to download the full stream from service // then await assertOpenResponseProcessors(0); } finally { controller.abort(); } }); - - function KBperSecond(kilobytes) { - return 1000 * kilobytes; - } }); it('should serve generated client-config.json', async () => { From a65633508c2df3aba709bdd49f5e6afaf91c5da0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 08:18:02 +0000 Subject: [PATCH 06/30] wip --- files/nginx/odk.conf.template | 4 ++-- test/nginx/mock-http-server/index.js | 8 +++++--- test/nginx/src/mocha/nginx.spec.js | 28 +++++++++++++++++----------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 05d5c906b..065539a3e 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -216,8 +216,8 @@ server { proxy_buffering on; proxy_read_timeout 2m; proxy_temp_path /tmp; - proxy_max_temp_file_size 64m; - proxy_temp_file_write_size 64m; + proxy_max_temp_file_size 256m; + proxy_temp_file_write_size 256m; proxy_buffers 64 64k; proxy_buffer_size 64k; proxy_busy_buffers_size 128k; diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index 83823d42e..e4f5d0e17 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -36,7 +36,9 @@ app.get('/reset', (req, res) => { res.json('OK'); }); -app.get('/v1/25MB.csv', (req, res) => { +app.get('/v1/100MB.csv', (req, res) => { + const csvSizeBytes = 100_000_000; + // TODO confirm from IRL what headers should be set, e.g. // content-type // content-length @@ -48,7 +50,7 @@ app.get('/v1/25MB.csv', (req, res) => { let rowCount = 0; let written = 0; - const batchSize = Math.pow(2, 16); + const batchSize = Math.pow(2, 18); const header = Buffer.from('row_number,timestamp,random-number\n', 'utf8'); written += header.byteLength; @@ -69,7 +71,7 @@ app.get('/v1/25MB.csv', (req, res) => { } // TODO up this to 100MiB - const randomStream = Readable.from(generateCsv(25_000_000)); + const randomStream = Readable.from(generateCsv(csvSizeBytes)); randomStream.pipe(res); req.on('close', () => { console.log('req.closed; destroying randomStream'); diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index cc0e94139..80a4b72e6 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -438,19 +438,21 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); describe.only('response buffering', () => { - async function assertOpenResponseProcessors(expectedCount) { + async function getOpenProcessorCount() { const res = await request(`http://localhost:8383/open-processor-count`); assert.isTrue(res.ok); const body = await res.text(); console.log(`[open-processor-count: ${body}]`); - const actualCount = Number(body); - - assert.equal(actualCount, expectedCount); + return Number(body); } it('should buffer responses in nginx, not backend services', async function() { - this.timeout(20_000); // FIXME hopefully remove this + // NOTE the final check should pass before the test timeout. If it's taking longer, + // especially significantly longer, that implies that there is back-pressure on the + // NodeJS server. This would imply that nginx buffering is not working correctly. + // !!! ONLY CHANGE THIS TIMEOUT IF THE ABOVE COMMENT IS WELL UNDERSTOOD !!! + this.timeout(5_000); let controller; @@ -460,7 +462,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward const { signal } = controller; // when - const res = await apiFetch('/v1/25MB.csv', { signal }); + const res = await apiFetch('/v1/100MB.csv', { signal }); const reader = res.body.getReader(); const { done, value } = await reader.read(); console.log({ done, value }); @@ -468,12 +470,16 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward // then assert.equal(res.status, 200); // and - await assertOpenResponseProcessors(1); + assert.equal(await getOpenProcessorCount(), 1); - // when - await sleep(10_000); // should be long enough for nginx to download the full stream from service - // then - await assertOpenResponseProcessors(0); + let openProcessorCount; + do { + // when + await sleep(100); + openProcessorCount = await getOpenProcessorCount(); + + // then + } while(openProcessorCount !== 0); } finally { controller.abort(); } From b560f8751bcf9d39cfb45a6fb28934fb3639db9f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 08:32:43 +0000 Subject: [PATCH 07/30] wip --- files/nginx/odk.conf.template | 10 +--------- test/nginx/src/mocha/nginx.spec.js | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 065539a3e..4a5551c7b 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -200,8 +200,6 @@ server { # central-backend location ~ ^/v\d { - error_log /dev/stderr info; - proxy_hide_header Content-Security-Policy; add_header Content-Security-Policy $central_backend_csp always; @@ -213,14 +211,8 @@ server { # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; - proxy_buffering on; + proxy_buffering off; proxy_read_timeout 2m; - proxy_temp_path /tmp; - proxy_max_temp_file_size 256m; - proxy_temp_file_write_size 256m; - proxy_buffers 64 64k; - proxy_buffer_size 64k; - proxy_busy_buffers_size 128k; } location @blank.html { diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 80a4b72e6..a188cd78f 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -447,12 +447,28 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward return Number(body); } + async function untilOpenProcessorCountIs(expectedCount) { + while(await getOpenProcessorCount() !== expectedCount) { + await sleep(100); + } + } + + function withTimeout(ms, promise) { + let timeoutId; + const timeout = new Promise(resolve => timeoutId = setTimeout(resolve, ms)); + + return Promise + .race([ /*timeout,*/ promise ]) + .finally(() => clearTimeout(timeoutId)); + } + it('should buffer responses in nginx, not backend services', async function() { // NOTE the final check should pass before the test timeout. If it's taking longer, // especially significantly longer, that implies that there is back-pressure on the // NodeJS server. This would imply that nginx buffering is not working correctly. // !!! ONLY CHANGE THIS TIMEOUT IF THE ABOVE COMMENT IS WELL UNDERSTOOD !!! - this.timeout(5_000); + const testTimeout = 5_000; + this.timeout(testTimeout); let controller; @@ -472,14 +488,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward // and assert.equal(await getOpenProcessorCount(), 1); - let openProcessorCount; - do { - // when - await sleep(100); - openProcessorCount = await getOpenProcessorCount(); - - // then - } while(openProcessorCount !== 0); + await withTimeout(testTimeout, untilOpenProcessorCountIs(0)); } finally { controller.abort(); } From 04f2b8ae829f067e9978a94dc72f18a1583d7f56 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:08:10 +0000 Subject: [PATCH 08/30] passing test --- files/nginx/odk.conf.template | 10 ++++++- test/nginx/mock-http-server/index.js | 11 ++++---- test/nginx/src/mocha/nginx.spec.js | 41 +++++++++++++++------------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 4a5551c7b..065539a3e 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -200,6 +200,8 @@ server { # central-backend location ~ ^/v\d { + error_log /dev/stderr info; + proxy_hide_header Content-Security-Policy; add_header Content-Security-Policy $central_backend_csp always; @@ -211,8 +213,14 @@ server { # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; - proxy_buffering off; + proxy_buffering on; proxy_read_timeout 2m; + proxy_temp_path /tmp; + proxy_max_temp_file_size 256m; + proxy_temp_file_write_size 256m; + proxy_buffers 64 64k; + proxy_buffer_size 64k; + proxy_busy_buffers_size 128k; } location @blank.html { diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index e4f5d0e17..3f5641bce 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -7,6 +7,7 @@ const log = (...args) => console.log('[mock-http-server]', ...args); const requests = []; let openProcessorCount = 0; +let completedProcessorCount = 0; const app = express(); app.set('case sensitive routing', true); @@ -33,6 +34,7 @@ app.get('/request-log', (req, res) => res.json(requests)); app.get('/reset', (req, res) => { requests.length = 0; openProcessorCount = 0; + completedProcessorCount = 0; res.json('OK'); }); @@ -65,25 +67,22 @@ app.get('/v1/100MB.csv', (req, res) => { bufpos += batch.write(line, bufpos, 'utf8'); } written += batchSize; - console.log('written:', written); yield batch; } + + ++completedProcessorCount; } // TODO up this to 100MiB const randomStream = Readable.from(generateCsv(csvSizeBytes)); randomStream.pipe(res); req.on('close', () => { - console.log('req.closed; destroying randomStream'); randomStream.destroy(); --openProcessorCount; }); }); app.get('/open-processor-count', (req, res) => { - console.log(` - /open-processor-count called; count: ${openProcessorCount} - `); - res.send(openProcessorCount); + res.send({ openProcessorCount, completedProcessorCount }); }); app.get('/v1/reflect-headers', (req, res) => res.json(req.headers)); diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index a188cd78f..9c12142be 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -441,25 +441,29 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward async function getOpenProcessorCount() { const res = await request(`http://localhost:8383/open-processor-count`); assert.isTrue(res.ok); - - const body = await res.text(); - console.log(`[open-processor-count: ${body}]`); - return Number(body); + return await res.json(); } - async function untilOpenProcessorCountIs(expectedCount) { - while(await getOpenProcessorCount() !== expectedCount) { - await sleep(100); - } - } - - function withTimeout(ms, promise) { + async function untilOpenProcessorCountIs({ timeout, ...expected }) { let timeoutId; - const timeout = new Promise(resolve => timeoutId = setTimeout(resolve, ms)); + try { + let timedOut; + timeoutId = setTimeout(() => { timedOut = true; }, timeout); + + while(true) { + const { openProcessorCount, completedProcessorCount } = await getOpenProcessorCount(); + if(openProcessorCount === expected.openProcessorCount && + completedProcessorCount === expected.completedProcessorCount) { + break; + } + + if(timedOut) throw new Error(`Timeout of ${timeout} ms exceeded.`); - return Promise - .race([ /*timeout,*/ promise ]) - .finally(() => clearTimeout(timeoutId)); + await sleep(100); + } + } finally { + clearTimeout(timeoutId); + } } it('should buffer responses in nginx, not backend services', async function() { @@ -480,15 +484,14 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward // when const res = await apiFetch('/v1/100MB.csv', { signal }); const reader = res.body.getReader(); - const { done, value } = await reader.read(); - console.log({ done, value }); + await reader.read(); // then assert.equal(res.status, 200); // and - assert.equal(await getOpenProcessorCount(), 1); + assert.deepEqual(await getOpenProcessorCount(), { openProcessorCount:1, completedProcessorCount:0 }); - await withTimeout(testTimeout, untilOpenProcessorCountIs(0)); + await untilOpenProcessorCountIs({ timeout:testTimeout, openProcessorCount:0, completedProcessorCount:1 }); } finally { controller.abort(); } From 9571e08f99892b447ddaa736863327e56c45a4ab Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:09:29 +0000 Subject: [PATCH 09/30] reduce logging --- files/nginx/odk.conf.template | 2 -- 1 file changed, 2 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 065539a3e..c686ed0c4 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -200,8 +200,6 @@ server { # central-backend location ~ ^/v\d { - error_log /dev/stderr info; - proxy_hide_header Content-Security-Policy; add_header Content-Security-Policy $central_backend_csp always; From 3282f95e9a48454c773c3b6854301ac43bb73b43 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:10:35 +0000 Subject: [PATCH 10/30] enable all tests --- test/nginx/src/mocha/nginx.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 9c12142be..76e0adf16 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -290,7 +290,7 @@ describe('nginx config', () => { resetBackendMock(), ])); - describe.only('SSL_TYPE=selfsign', () => { + describe('SSL_TYPE=selfsign', () => { const { fetchHttp, fetchHttp6, fetchHttps, fetchHttps6 } = fetchFunctionsForPorts(9000, 9001); it('HTTP should forward to HTTPS', async () => { @@ -363,7 +363,7 @@ describe('nginx config', () => { }); }); - describe.skip('SSL_TYPE=upstream', () => { + describe('SSL_TYPE=upstream', () => { const { fetchHttp, fetchHttp6, fetchHttps, fetchHttps6 } = fetchFunctionsForPorts(10000, 10001); it('should not respond to HTTPS requests (IPv4)', async () => { @@ -437,7 +437,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); }); - describe.only('response buffering', () => { + describe('response buffering', () => { async function getOpenProcessorCount() { const res = await request(`http://localhost:8383/open-processor-count`); assert.isTrue(res.ok); From bb546576cd5fcf48ec4f0e493a3c07092177d0e5 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:12:38 +0000 Subject: [PATCH 11/30] function order --- test/nginx/src/mocha/nginx.spec.js | 56 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 76e0adf16..987583617 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -438,34 +438,6 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); describe('response buffering', () => { - async function getOpenProcessorCount() { - const res = await request(`http://localhost:8383/open-processor-count`); - assert.isTrue(res.ok); - return await res.json(); - } - - async function untilOpenProcessorCountIs({ timeout, ...expected }) { - let timeoutId; - try { - let timedOut; - timeoutId = setTimeout(() => { timedOut = true; }, timeout); - - while(true) { - const { openProcessorCount, completedProcessorCount } = await getOpenProcessorCount(); - if(openProcessorCount === expected.openProcessorCount && - completedProcessorCount === expected.completedProcessorCount) { - break; - } - - if(timedOut) throw new Error(`Timeout of ${timeout} ms exceeded.`); - - await sleep(100); - } - } finally { - clearTimeout(timeoutId); - } - } - it('should buffer responses in nginx, not backend services', async function() { // NOTE the final check should pass before the test timeout. If it's taking longer, // especially significantly longer, that implies that there is back-pressure on the @@ -496,6 +468,34 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward controller.abort(); } }); + + async function getOpenProcessorCount() { + const res = await request(`http://localhost:8383/open-processor-count`); + assert.isTrue(res.ok); + return await res.json(); + } + + async function untilOpenProcessorCountIs({ timeout, ...expected }) { + let timeoutId; + try { + let timedOut; + timeoutId = setTimeout(() => { timedOut = true; }, timeout); + + while(true) { + const { openProcessorCount, completedProcessorCount } = await getOpenProcessorCount(); + if(openProcessorCount === expected.openProcessorCount && + completedProcessorCount === expected.completedProcessorCount) { + break; + } + + if(timedOut) throw new Error(`Timeout of ${timeout} ms exceeded.`); + + await sleep(100); + } + } finally { + clearTimeout(timeoutId); + } + } }); it('should serve generated client-config.json', async () => { From e90b350bfc16df20e01460131ff0c5ae7706bfc5 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:14:43 +0000 Subject: [PATCH 12/30] remove unused todo --- test/nginx/mock-http-server/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index 3f5641bce..cfe0e9d51 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -73,7 +73,6 @@ app.get('/v1/100MB.csv', (req, res) => { ++completedProcessorCount; } - // TODO up this to 100MiB const randomStream = Readable.from(generateCsv(csvSizeBytes)); randomStream.pipe(res); req.on('close', () => { From 4f992d67e46114d0d53bf775be26b141665c6b28 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:17:51 +0000 Subject: [PATCH 13/30] update comment --- test/nginx/src/mocha/nginx.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 987583617..8d43da81b 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -439,9 +439,9 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward describe('response buffering', () => { it('should buffer responses in nginx, not backend services', async function() { - // NOTE the final check should pass before the test timeout. If it's taking longer, - // especially significantly longer, that implies that there is back-pressure on the - // NodeJS server. This would imply that nginx buffering is not working correctly. + // NOTE the final check should pass before the test times out. If timeout + // is exceeded, there may be back-pressure on the NodeJS server. This + // would imply that nginx buffering is not working correctly. // !!! ONLY CHANGE THIS TIMEOUT IF THE ABOVE COMMENT IS WELL UNDERSTOOD !!! const testTimeout = 5_000; this.timeout(testTimeout); From e949a7309e113c54f3dee507e59ef3ff67d89fdf Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:20:42 +0000 Subject: [PATCH 14/30] remove one thing --- files/nginx/odk.conf.template | 1 - 1 file changed, 1 deletion(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index c686ed0c4..26ebfb4f2 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -213,7 +213,6 @@ server { proxy_request_buffering on; proxy_buffering on; proxy_read_timeout 2m; - proxy_temp_path /tmp; proxy_max_temp_file_size 256m; proxy_temp_file_write_size 256m; proxy_buffers 64 64k; From 9f9fa4b9d0dcb92bc4ee7c72da30caf5eb415f9c Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:21:23 +0000 Subject: [PATCH 15/30] remove more new settings --- files/nginx/odk.conf.template | 3 --- 1 file changed, 3 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 26ebfb4f2..5137ef46b 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -215,9 +215,6 @@ server { proxy_read_timeout 2m; proxy_max_temp_file_size 256m; proxy_temp_file_write_size 256m; - proxy_buffers 64 64k; - proxy_buffer_size 64k; - proxy_busy_buffers_size 128k; } location @blank.html { From 9bc1a6860a32b3f39b6c771ef005807e5dca3abc Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:21:54 +0000 Subject: [PATCH 16/30] reduce buffer sizes --- files/nginx/odk.conf.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 5137ef46b..3ab75a1cf 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -213,8 +213,8 @@ server { proxy_request_buffering on; proxy_buffering on; proxy_read_timeout 2m; - proxy_max_temp_file_size 256m; - proxy_temp_file_write_size 256m; + proxy_max_temp_file_size 100m; + proxy_temp_file_write_size 100m; } location @blank.html { From dd05e6ce2730bab806c1607f64a125f497a27d8a Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:24:57 +0000 Subject: [PATCH 17/30] less settings --- files/nginx/odk.conf.template | 1 - 1 file changed, 1 deletion(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 3ab75a1cf..fbc77f10b 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -213,7 +213,6 @@ server { proxy_request_buffering on; proxy_buffering on; proxy_read_timeout 2m; - proxy_max_temp_file_size 100m; proxy_temp_file_write_size 100m; } From e73f72fac2cb4e791c868109d2fa32b982676c5b Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:25:37 +0000 Subject: [PATCH 18/30] less settings --- files/nginx/odk.conf.template | 1 - 1 file changed, 1 deletion(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index fbc77f10b..f6ea298ed 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -213,7 +213,6 @@ server { proxy_request_buffering on; proxy_buffering on; proxy_read_timeout 2m; - proxy_temp_file_write_size 100m; } location @blank.html { From ec72af4be3b9774ecb35db19ee626c8df6eb984f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 6 Jul 2026 09:30:00 +0000 Subject: [PATCH 19/30] add TODO --- test/nginx/mock-http-server/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index cfe0e9d51..412d3b212 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -38,6 +38,7 @@ app.get('/reset', (req, res) => { res.json('OK'); }); +// TODO make this path more realistic app.get('/v1/100MB.csv', (req, res) => { const csvSizeBytes = 100_000_000; From 7e4361dc4b3573bbe0795af78fad33139512abc8 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 7 Jul 2026 04:29:41 +0000 Subject: [PATCH 20/30] update ci from next --- .github/workflows/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2aac7df2f..f0d99ef8e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -162,6 +162,13 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + - name: Extract FRONTEND_VERSION + run: | + touch .env + echo "FRONTEND_VERSION=$( + docker compose config --format json | jq -r .services.nginx.build.args.FRONTEND_VERSION + )" >> "$GITHUB_ENV" + - name: Build and push ${{ matrix.image }} Docker image uses: docker/build-push-action@v7 with: @@ -171,3 +178,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: 'linux/amd64,linux/arm64' + build-args: | + FRONTEND_BUILD_MODE=fetch + FRONTEND_VERSION=${{ env.FRONTEND_VERSION }} From 69d447d025b12c6786523e1735546fdc7df8841c Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 7 Jul 2026 04:32:57 +0000 Subject: [PATCH 21/30] manually merge changes from next --- test/nginx/mock-http-server/index.js | 8 ++++---- test/nginx/mock-sentry/index.js | 4 ++-- test/nginx/setup-tests.sh | 4 ++-- test/nginx/src/lib.js | 4 ++-- test/nginx/src/mocha/nginx.spec.js | 6 +++--- test/package.json | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index 412d3b212..b10b51a98 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -29,9 +29,9 @@ app.use('/-/', (req, res, next) => { next(); }); -app.get('/health', (req, res) => res.send('OK')); -app.get('/request-log', (req, res) => res.json(requests)); -app.get('/reset', (req, res) => { +app.get('/__mock_http_server/health', (req, res) => res.send('OK')); +app.get('/__mock_http_server/request-log', (req, res) => res.json(requests)); +app.get('/__mock_http_server/reset', (req, res) => { requests.length = 0; openProcessorCount = 0; completedProcessorCount = 0; @@ -81,7 +81,7 @@ app.get('/v1/100MB.csv', (req, res) => { --openProcessorCount; }); }); -app.get('/open-processor-count', (req, res) => { +app.get('/__mock_http_server/open-processor-count', (req, res) => { res.send({ openProcessorCount, completedProcessorCount }); }); diff --git a/test/nginx/mock-sentry/index.js b/test/nginx/mock-sentry/index.js index c119c595d..d72a75569 100644 --- a/test/nginx/mock-sentry/index.js +++ b/test/nginx/mock-sentry/index.js @@ -27,8 +27,8 @@ app.use(express.json({ 'application/reports+json', // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Security-Policy/report-to#violation_report_syntax ], })); -app.get('/event-log', (req, res) => res.json(events)); -app.get('/reset', (req, res) => { +app.get('/__mock_sentry/event-log', (req, res) => res.json(events)); +app.get('/__mock_sentry/reset', (req, res) => { events.length = 0; res.json('OK'); }); diff --git a/test/nginx/setup-tests.sh b/test/nginx/setup-tests.sh index 241a07d53..44774afaa 100755 --- a/test/nginx/setup-tests.sh +++ b/test/nginx/setup-tests.sh @@ -31,9 +31,9 @@ log "Starting test services..." docker_compose up --build --detach log "Waiting for mock backend..." -wait_for_http_response 5 localhost:8383/health 200 +wait_for_http_response 5 localhost:8383/__mock_http_server/health 200 log "Waiting for mock enketo..." -wait_for_http_response 5 localhost:8005/health 200 +wait_for_http_response 5 localhost:8005/__mock_http_server/health 200 log "Waiting for nginx..." wait_for_http_response 5 localhost:9000 421 diff --git a/test/nginx/src/lib.js b/test/nginx/src/lib.js index b7484bbc6..d96a2de2d 100644 --- a/test/nginx/src/lib.js +++ b/test/nginx/src/lib.js @@ -14,7 +14,7 @@ module.exports = { }; async function assertSentryReceived(...expectedRequests) { - const { status, body } = await requestSentryMock({ path:'/event-log' }); + const { status, body } = await requestSentryMock({ path:'/__mock_sentry/event-log' }); assert.equal(status, 200); const actual = JSON.parse(body); @@ -28,7 +28,7 @@ async function assertSentryReceived(...expectedRequests) { } async function resetSentryMock() { - const res = await requestSentryMock({ path:'/reset' }); + const res = await requestSentryMock({ path:'/__mock_sentry/reset' }); assert.equal(res.status, 200); } diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 8d43da81b..6fa748397 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -470,7 +470,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); async function getOpenProcessorCount() { - const res = await request(`http://localhost:8383/open-processor-count`); + const res = await request(`http://localhost:8383/__mock_http_server/open-processor-count`); assert.isTrue(res.ok); return await res.json(); } @@ -1224,7 +1224,7 @@ function assertBackendReceived(...expectedRequests) { } async function assertMockHttpReceived(port, expectedRequests) { - const res = await request(`http://localhost:${port}/request-log`); + const res = await request(`http://localhost:${port}/__mock_http_server/request-log`); assert.isTrue(res.ok); assert.deepEqual(expectedRequests, await res.json()); } @@ -1238,7 +1238,7 @@ function resetBackendMock() { } async function resetMock(port) { - const res = await request(`http://localhost:${port}/reset`); + const res = await request(`http://localhost:${port}/__mock_http_server/reset`); assert.isTrue(res.ok); } diff --git a/test/package.json b/test/package.json index f372bc0ae..9e46aa6b2 100644 --- a/test/package.json +++ b/test/package.json @@ -4,7 +4,7 @@ "lint": "ESLINT_USE_FLAT_CONFIG=false npx eslint --cache .", "test:github-actions": "mocha ./github-actions.spec.js", "test:nginx": "npm run test:nginx:mocha && npm run test:nginx:playwright", - "test:nginx:mocha": "NODE_TLS_REJECT_UNAUTHORIZED=0 mocha ./nginx/src/mocha", + "test:nginx:mocha": "NODE_TLS_REJECT_UNAUTHORIZED=0 mocha './nginx/src/mocha/**/*.spec.js' --require ./nginx/src/mocha/mocha.setup.js", "test:nginx:playwright": "NODE_TLS_REJECT_UNAUTHORIZED=0 playwright test", "test:service": "mocha ./service.spec.js", "test": "npm run lint && npm run test:github-actions && npm run test:nginx && npm run test:service" From 034cbe6dbcb1f09cff7696575584fa9ed7c05736 Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:36:11 +0300 Subject: [PATCH 22/30] Apply suggestion from @alxndrsn --- test/nginx/src/mocha/nginx.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 6fa748397..91ecb2522 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -440,7 +440,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward describe('response buffering', () => { it('should buffer responses in nginx, not backend services', async function() { // NOTE the final check should pass before the test times out. If timeout - // is exceeded, there may be back-pressure on the NodeJS server. This + // is reached, there may be back-pressure on the NodeJS server. This // would imply that nginx buffering is not working correctly. // !!! ONLY CHANGE THIS TIMEOUT IF THE ABOVE COMMENT IS WELL UNDERSTOOD !!! const testTimeout = 5_000; From 135a005d2188ecefb1eb954366837c82565f3efc Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 7 Jul 2026 04:53:24 +0000 Subject: [PATCH 23/30] match all .csv paths --- test/nginx/mock-http-server/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index b10b51a98..650fb0f26 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -38,8 +38,7 @@ app.get('/__mock_http_server/reset', (req, res) => { res.json('OK'); }); -// TODO make this path more realistic -app.get('/v1/100MB.csv', (req, res) => { +app.get(new RegExp('^/v1/.*\\.csv$'), (req, res) => { const csvSizeBytes = 100_000_000; // TODO confirm from IRL what headers should be set, e.g. From 20766c15b3c53aa6ba594c1a83e594853eca6506 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 7 Jul 2026 04:56:35 +0000 Subject: [PATCH 24/30] use more realistic path --- test/nginx/mock-http-server/index.js | 2 +- test/nginx/src/mocha/nginx.spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index 650fb0f26..a79efb9ec 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -38,7 +38,7 @@ app.get('/__mock_http_server/reset', (req, res) => { res.json('OK'); }); -app.get(new RegExp('^/v1/.*\\.csv$'), (req, res) => { +app.get(new RegExp('^/v1/.*/100MB\\.csv$'), (req, res) => { const csvSizeBytes = 100_000_000; // TODO confirm from IRL what headers should be set, e.g. diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index 91ecb2522..c1d3ed913 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -454,7 +454,7 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward const { signal } = controller; // when - const res = await apiFetch('/v1/100MB.csv', { signal }); + const res = await apiFetch('/v1/projects/123/forms/some_form_id/attachments/100MB.csv', { signal }); const reader = res.body.getReader(); await reader.read(); From 1212289d0aa55b967eae70b0427671b513e5e788 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 7 Jul 2026 06:12:32 +0000 Subject: [PATCH 25/30] resolve todo --- test/nginx/mock-http-server/index.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index a79efb9ec..cc7f83664 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -41,10 +41,8 @@ app.get('/__mock_http_server/reset', (req, res) => { app.get(new RegExp('^/v1/.*/100MB\\.csv$'), (req, res) => { const csvSizeBytes = 100_000_000; - // TODO confirm from IRL what headers should be set, e.g. - // content-type - // content-length - // transfer-encoding + res.set('Content-Disposition', `attachment; filename="100MB.csv"; filename*=UTF-8''100MB.csv`); + res.set('Content-Type', 'text/csv; charset=utf-8'); ++openProcessorCount; From 3393e69719486a7ff0c313e0c0e3a8cd54c6dbcf Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Sat, 11 Jul 2026 07:11:30 +0000 Subject: [PATCH 26/30] remove outdated comment --- files/nginx/odk.conf.template | 1 - 1 file changed, 1 deletion(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index f6ea298ed..dcf6c47a1 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -209,7 +209,6 @@ server { proxy_pass http://service:8383; proxy_redirect off; - # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; proxy_buffering on; proxy_read_timeout 2m; From 10fcce320bd7f54bbbe0e490195b1865b45ccd28 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 21 Jul 2026 05:50:44 +0000 Subject: [PATCH 27/30] wip --- test/nginx/mock-http-server/index.js | 16 +++++++++------- test/nginx/src/mocha/nginx.spec.js | 21 ++++++++++++++++++--- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index cc7f83664..b88fe56a1 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -48,24 +48,26 @@ app.get(new RegExp('^/v1/.*/100MB\\.csv$'), (req, res) => { async function* generateCsv(targetByteLength) { let rowCount = 0; - let written = 0; + let totalWritten = 0; const batchSize = Math.pow(2, 18); const header = Buffer.from('row_number,timestamp,random-number\n', 'utf8'); - written += header.byteLength; + totalWritten += header.byteLength; yield header; - while(written < targetByteLength) { + while(totalWritten < targetByteLength) { await new Promise(resolve => setTimeout(resolve, 1)); + const batch = Buffer.allocUnsafe(batchSize); let bufpos = 0; - while(bufpos < batchSize) { + while(bufpos < batchSize && totalWritten < targetByteLength) { const line = `${++rowCount},${new Date().toISOString()},${Math.random()}\n`; - bufpos += batch.write(line, bufpos, 'utf8'); + const writtenNow = batch.write(line.substring(0, targetByteLength-totalWritten), bufpos, 'utf8'); + bufpos += writtenNow; + totalWritten += writtenNow; } - written += batchSize; - yield batch; + yield batch.subarray(0, bufpos); } ++completedProcessorCount; diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index dacb7fc9e..a17869241 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -455,15 +455,30 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward // when const res = await apiFetch('/v1/projects/123/forms/some_form_id/attachments/100MB.csv', { signal }); - const reader = res.body.getReader(); - await reader.read(); - // then assert.equal(res.status, 200); + + // when + const reader = res.body.getReader(); + const initialRead = await reader.read(); + let bytesRead = initialRead.value.length; + // then + assert.isFalse(initialRead.done); + assert.isAtMost(bytesRead, 16_384); + assert.equal(new TextDecoder('utf8').decode(initialRead.value).split('\n', 1)[0], 'row_number,timestamp,random-number'); // and assert.deepEqual(await getOpenProcessorCount(), { openProcessorCount:1, completedProcessorCount:0 }); + // when await untilOpenProcessorCountIs({ timeout:testTimeout, openProcessorCount:0, completedProcessorCount:1 }); + // and + while(true) { + const { done, value } = await reader.read(); + if(done) break; + bytesRead += value.length; + } + // then + assert.equal(bytesRead, 100_000_000); } finally { controller.abort(); } From a493365bb3021331247f3d5eb8229c8caf20f8ac Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 21 Jul 2026 06:01:17 +0000 Subject: [PATCH 28/30] write exact size --- test/nginx/mock-http-server/index.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index b88fe56a1..707878f2e 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -59,15 +59,15 @@ app.get(new RegExp('^/v1/.*/100MB\\.csv$'), (req, res) => { while(totalWritten < targetByteLength) { await new Promise(resolve => setTimeout(resolve, 1)); - const batch = Buffer.allocUnsafe(batchSize); + const batch = Buffer.allocUnsafe(Math.min(batchSize, targetByteLength - totalWritten)); let bufpos = 0; - while(bufpos < batchSize && totalWritten < targetByteLength) { + while(bufpos < batch.length) { const line = `${++rowCount},${new Date().toISOString()},${Math.random()}\n`; - const writtenNow = batch.write(line.substring(0, targetByteLength-totalWritten), bufpos, 'utf8'); - bufpos += writtenNow; - totalWritten += writtenNow; + const bytesWritten = batch.write(line, bufpos, batch.length-bufpos, 'utf8'); + bufpos += bytesWritten; + totalWritten += bytesWritten; } - yield batch.subarray(0, bufpos); + yield batch; } ++completedProcessorCount; From 8f25a345cd22ab5c9432ca020836d0ddbc4bb3ba Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 23 Jul 2026 09:39:44 +0000 Subject: [PATCH 29/30] remove comment --- files/nginx/odk.conf.template | 3 ++- test/nginx/src/mocha/nginx.spec.js | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index dcf6c47a1..4a5551c7b 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -209,8 +209,9 @@ server { proxy_pass http://service:8383; proxy_redirect off; + # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; - proxy_buffering on; + proxy_buffering off; proxy_read_timeout 2m; } diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index a17869241..8b1e52771 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -439,10 +439,6 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward describe('response buffering', () => { it('should buffer responses in nginx, not backend services', async function() { - // NOTE the final check should pass before the test times out. If timeout - // is reached, there may be back-pressure on the NodeJS server. This - // would imply that nginx buffering is not working correctly. - // !!! ONLY CHANGE THIS TIMEOUT IF THE ABOVE COMMENT IS WELL UNDERSTOOD !!! const testTimeout = 5_000; this.timeout(testTimeout); From 2a80b532498593115cbe605a11c8d43ad597a3f0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 23 Jul 2026 09:43:28 +0000 Subject: [PATCH 30/30] reintroduce nginx config change --- files/nginx/odk.conf.template | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 4a5551c7b..dcf6c47a1 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -209,9 +209,8 @@ server { proxy_pass http://service:8383; proxy_redirect off; - # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; - proxy_buffering off; + proxy_buffering on; proxy_read_timeout 2m; }