From b2d809c0ca8b1230eaa1802f07a5fc636724bd5f Mon Sep 17 00:00:00 2001 From: agpuperson <81822275+Mr-Tech-13@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:11:31 -0500 Subject: [PATCH 01/13] Update nvidia-gb.ts Update Selectors, add 5090/5080 FE --- src/store/model/nvidia-gb.ts | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/store/model/nvidia-gb.ts b/src/store/model/nvidia-gb.ts index 52a460939d..f9cadf61fb 100644 --- a/src/store/model/nvidia-gb.ts +++ b/src/store/model/nvidia-gb.ts @@ -4,12 +4,12 @@ export const NvidiaGB: Store = { currency: '£', labels: { inStock: { - container: '.buy', - text: ['add to cart', 'buy now'], + container: 'span.buy-link-atc', + text: ['Buy Now', 'Add to Cart'] }, outOfStock: { - container: '.buy', - text: ['out of stock'], + container: 'span.buy-link-atc', + text: ['Out of Stock'], }, }, links: [ @@ -55,7 +55,25 @@ export const NvidiaGB: Store = { series: '3090', url: 'https://shop.nvidia.com/en-gb/geforce/store/gpu/?page=1&limit=9&locale=en-gb&category=GPU&gpu=RTX%203090&manufacturer=NVIDIA', }, + { + brand: 'asus', + model: 'dual', + series: '4070', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&manufacturer=ASUS&gpu=RTX%204070&', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5090', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5080', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&manufacturer=NVIDIA&gpu=RTX%205080&', + }, ], name: 'nvidia-gb', country: 'UK', -}; +}; \ No newline at end of file From 03564dd6f5922c9df2b11bd1826e3534caf9ba66 Mon Sep 17 00:00:00 2001 From: agpuperson <81822275+Mr-Tech-13@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:32:12 -0500 Subject: [PATCH 02/13] Update nvidia GB and DE --- src/store/model/nvidia-de.ts | 26 +++++++++++++++++++++++--- src/store/model/nvidia-gb.ts | 11 ++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/store/model/nvidia-de.ts b/src/store/model/nvidia-de.ts index e8de308d1c..025d34dfee 100644 --- a/src/store/model/nvidia-de.ts +++ b/src/store/model/nvidia-de.ts @@ -4,13 +4,13 @@ export const NvidiaDE: Store = { currency: '€', labels: { inStock: { - container: '.buy', + container: 'span.buy-link-atc', text: ['In den Einkaufswagen', 'JETZT KAUFEN'], }, outOfStock: { - container: '.buy', + container: 'span.buy-link-atc', text: ['DERZEIT NICHT VERFÜGBAR'], - }, + }, }, links: [ { @@ -55,7 +55,27 @@ export const NvidiaDE: Store = { series: '3090', url: 'https://shop.nvidia.com/de-de/geforce/store/gpu/?page=1&limit=9&locale=de-de&category=GPU&gpu=RTX%203090&manufacturer=NVIDIA', }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5080', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5090', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + }, ], name: 'nvidia-de', country: 'DE', }; +/* Copy Paste Template + { + brand: 'nvidia', + model: 'founders edition', + series: '', + url: '', + }, +*/ diff --git a/src/store/model/nvidia-gb.ts b/src/store/model/nvidia-gb.ts index f9cadf61fb..a18ca704d8 100644 --- a/src/store/model/nvidia-gb.ts +++ b/src/store/model/nvidia-gb.ts @@ -76,4 +76,13 @@ export const NvidiaGB: Store = { ], name: 'nvidia-gb', country: 'UK', -}; \ No newline at end of file +}; + +/* Copy Paste Template + { + brand: 'nvidia', + model: 'founders edition', + series: '', + url: '', + }, +*/ \ No newline at end of file From ea78b9422e4f938c0e52cdaf3a7df4a7672e4e15 Mon Sep 17 00:00:00 2001 From: agpuperson <81822275+Mr-Tech-13@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:41:14 -0500 Subject: [PATCH 03/13] Update nvidia-fr.ts --- src/store/model/nvidia-fr.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/store/model/nvidia-fr.ts b/src/store/model/nvidia-fr.ts index b236c50446..11e9ad1ac9 100644 --- a/src/store/model/nvidia-fr.ts +++ b/src/store/model/nvidia-fr.ts @@ -4,11 +4,11 @@ export const NvidiaFR: Store = { currency: '€', labels: { inStock: { - container: '.buy', - text: ['ajouter au panier', 'acheter maintenant'], + container: 'span.buy-link-atc', + text: ['Acheter maintenant'], }, outOfStock: { - container: '.buy', + container: 'span.buy-link-atc', text: ['RUPTURE DE STOCK'], }, }, @@ -55,7 +55,21 @@ export const NvidiaFR: Store = { series: '3090', url: 'https://shop.nvidia.com/fr-fr/geforce/store/gpu/?page=1&limit=9&locale=fr-fr&category=GPU&gpu=RTX%203090&manufacturer=NVIDIA', }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5090', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%204090&manufacturer=MSI', + }, ], name: 'nvidia-fr', country: 'FR', }; +/* Copy Paste Template + { + brand: 'nvidia', + model: 'founders edition', + series: '', + url: '', + }, +*/ \ No newline at end of file From 4cc67417a7075781d8f1a5adc26b217bcacc136b Mon Sep 17 00:00:00 2001 From: agpuperson <81822275+Mr-Tech-13@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:42:06 -0500 Subject: [PATCH 04/13] Update nvidia-fr.ts --- src/store/model/nvidia-fr.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/store/model/nvidia-fr.ts b/src/store/model/nvidia-fr.ts index 11e9ad1ac9..15aa6544a4 100644 --- a/src/store/model/nvidia-fr.ts +++ b/src/store/model/nvidia-fr.ts @@ -59,7 +59,13 @@ export const NvidiaFR: Store = { brand: 'nvidia', model: 'founders edition', series: '5090', - url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%204090&manufacturer=MSI', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5080', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', }, ], name: 'nvidia-fr', From 05ed4909832bbe6cdb587d44831558daecb58002 Mon Sep 17 00:00:00 2001 From: agpuperson Date: Thu, 26 Mar 2026 19:51:26 -0400 Subject: [PATCH 05/13] Add live matrix UI with persisted settings and bot controls - replace the old web UI with a live matrix dashboard showing store x series status - add runtime status tracking for idle/checking/in stock/out of stock/error cells - expose matrix, settings, and control APIs from the web server - add top-right filter menus for stores, series, and models - persist filter changes back to the active dotenv file - add a gear/settings editor for the full active dotenv file - add in-page bot restart support without tearing down the web server - add configurable lookup concurrency via LOOKUP_THREADS - add randomized store/product ordering via RANDOMIZE_LOOKUP_ORDER - update matrix behavior so the latest status event wins and selected series render as idle when appropriate --- dotenv-example | 2 + src/config.ts | 44 +- src/index.ts | 82 ++- src/runtime-control.ts | 14 + src/store/lookup.ts | 290 +++++---- src/web/index.ts | 71 +++ src/web/settings.ts | 119 ++++ src/web/status.ts | 169 ++++++ web/index.html | 1302 ++++++++++++++++++++++++++++++++++------ 9 files changed, 1737 insertions(+), 356 deletions(-) create mode 100644 src/runtime-control.ts create mode 100644 src/web/settings.ts create mode 100644 src/web/status.ts diff --git a/dotenv-example b/dotenv-example index f01ae17bd5..e5433e0d42 100644 --- a/dotenv-example +++ b/dotenv-example @@ -74,6 +74,8 @@ IN_STOCK_WAIT_TIME= INCOGNITO= LOG_LEVEL= LOW_BANDWIDTH= +LOOKUP_THREADS=1 +RANDOMIZE_LOOKUP_ORDER=true MAX_PRICE_SERIES_3060= MAX_PRICE_SERIES_3060TI= MAX_PRICE_SERIES_3070= diff --git a/src/config.ts b/src/config.ts index a8211987c6..7dac3cfcfa 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,24 +3,36 @@ import {existsSync, readFileSync} from 'fs'; import path from 'path'; import {banner} from './banner'; -if (process.env.npm_config_conf) { - if ( - existsSync(path.resolve(__dirname, '../../' + process.env.npm_config_conf)) - ) { - dotenv.config({ - path: path.resolve(__dirname, '../../' + process.env.npm_config_conf), - }); - } else { - dotenv.config({path: path.resolve(__dirname, '../../.env')}); +export function getActiveConfigPath() { + if (process.env.npm_config_conf) { + const configuredPath = path.resolve( + __dirname, + '../../' + process.env.npm_config_conf + ); + if (existsSync(configuredPath)) { + return configuredPath; + } + + return path.resolve(__dirname, '../../.env'); } -} else if (existsSync(path.resolve(__dirname, '../../dotenv'))) { - dotenv.config({path: path.resolve(__dirname, '../../dotenv')}); -} else if (existsSync(path.resolve(__dirname, '../dotenv'))) { - dotenv.config({path: path.resolve(__dirname, '../dotenv')}); -} else { - dotenv.config({path: path.resolve(__dirname, '../../.env')}); + + const rootDotenv = path.resolve(__dirname, '../../dotenv'); + if (existsSync(rootDotenv)) { + return rootDotenv; + } + + const buildDotenv = path.resolve(__dirname, '../dotenv'); + if (existsSync(buildDotenv)) { + return buildDotenv; + } + + return path.resolve(__dirname, '../../.env'); } +export const activeConfigPath = getActiveConfigPath(); + +dotenv.config({path: activeConfigPath}); + console.info( banner.render( envOrBoolean(process.env.ASCII_BANNER, false), @@ -414,6 +426,8 @@ const nvidia = { const page = { height: 1080, inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME), + lookupThreads: envOrNumber(process.env.LOOKUP_THREADS, 1), + randomizeLookupOrder: envOrBoolean(process.env.RANDOMIZE_LOOKUP_ORDER, false), screenshot: envOrBoolean(process.env.SCREENSHOT), screenshotDir: envOrString(process.env.SCREENSHOT_DIR, 'screenshots'), timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000), diff --git a/src/index.ts b/src/index.ts index a140694739..b5fafe9b30 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,31 +6,46 @@ import {getSleepTime} from './util'; import {logger} from './logger'; import {storeList} from './store/model'; import {tryLookupAndLoop} from './store'; +import {setRestartBotHandler} from './runtime-control'; let browser: Browser | undefined; +let runGeneration = 0; + +function shuffleArray(items: T[]): T[] { + const shuffled = [...items]; + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; + } + + return shuffled; +} async function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } -/** - * Schedules a restart of the bot - */ -async function restartMain() { +async function scheduleRestart(generation: number) { if (config.restartTime > 0) { await sleep(config.restartTime); - await stop(); - loopMain(); + if (generation !== runGeneration) { + return; + } + + await restartBot(); } } -/** - * Starts the bot. - */ -async function main() { +async function startBot(startApiServer: boolean) { + const generation = ++runGeneration; browser = await launchBrowser(); + void scheduleRestart(generation); + + const stores = config.page.randomizeLookupOrder + ? shuffleArray([...storeList.values()]) + : [...storeList.values()]; - for (const store of storeList.values()) { + for (const store of stores) { logger.debug('store links', {meta: {links: store.links}}); if (store.setupAction !== undefined) { store.setupAction(browser); @@ -39,35 +54,43 @@ async function main() { setTimeout(tryLookupAndLoop, getSleepTime(store), browser, store); } - await startAPIServer(); + if (startApiServer) { + await startAPIServer(); + } } -async function stop() { - await stopAPIServer(); +async function stopBot() { + runGeneration++; if (browser) { - // Use temporary swap variable to avoid any race condition const browserTemporary = browser; browser = undefined; await browserTemporary.close(); } } +async function stop() { + await stopAPIServer(); + await stopBot(); +} + +export async function restartBot() { + logger.info('Restarting streetmerchant bot'); + await stopBot(); + await startBot(false); +} + async function stopAndExit() { await stop(); Process.exit(0); } -/** - * Will continually run until user interferes. - */ async function loopMain() { try { - restartMain(); - await main(); + await startBot(true); } catch (error: unknown) { logger.error( - '✖ something bad happened, resetting streetmerchant in 5 seconds', + '✖ something bad happened, resetting streetmerchant in 5 seconds', error ); setTimeout(loopMain, 5000); @@ -77,15 +100,11 @@ async function loopMain() { export async function launchBrowser(): Promise { const args: string[] = []; - // Skip Chromium Linux Sandbox - // https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox if (config.browser.isTrusted) { args.push('--no-sandbox'); args.push('--disable-setuid-sandbox'); } - // https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#tips - // https://stackoverflow.com/questions/48230901/docker-alpine-with-node-js-and-chromium-headless-puppeter-failed-to-launch-c if (config.docker) { args.push('--disable-dev-shm-usage'); args.push('--no-sandbox'); @@ -95,7 +114,6 @@ export async function launchBrowser(): Promise { config.browser.open = false; } - // Add the address of the proxy server if defined if (config.proxy.address) { args.push( `--proxy-server=${config.proxy.protocol}://${config.proxy.address}:${config.proxy.port}` @@ -103,11 +121,11 @@ export async function launchBrowser(): Promise { } if (args.length > 0) { - logger.info('ℹ puppeteer config: ', args); + logger.info('ℹ puppeteer config: ', args); } - await stop(); - const browser = await Puppeteer.launch({ + await stopBot(); + const launchedBrowser = await Puppeteer.launch({ args, defaultViewport: { height: config.page.height, @@ -116,11 +134,13 @@ export async function launchBrowser(): Promise { headless: config.browser.isHeadless, }); - config.browser.userAgent = await browser.userAgent(); + config.browser.userAgent = await launchedBrowser.userAgent(); - return browser; + return launchedBrowser; } +setRestartBotHandler(restartBot); + void loopMain(); process.on('SIGINT', stopAndExit); diff --git a/src/runtime-control.ts b/src/runtime-control.ts new file mode 100644 index 0000000000..d790992488 --- /dev/null +++ b/src/runtime-control.ts @@ -0,0 +1,14 @@ +let restartBotHandler: (() => Promise) | undefined; + +export function setRestartBotHandler(handler: () => Promise) { + restartBotHandler = handler; +} + +export async function restartBot() { + if (!restartBotHandler) { + throw new Error('Restart handler has not been initialized'); + } + + await restartBotHandler(); +} + diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 1c40cc613e..1448a3b789 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -28,11 +28,22 @@ import {handleCaptchaAsync} from './captcha-handler'; import useProxy from '@doridian/puppeteer-page-proxy'; import {promises as fs} from 'fs'; import path from 'path'; +import {markStatusChecking, markStatusResult} from '../web/status'; const inStock: Record = {}; const linkBuilderLastRunTimes: Record = {}; +function shuffleArray(items: T[]): T[] { + const shuffled = [...items]; + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; + } + + return shuffled; +} + function nextProxy(store: Store) { if (!store.proxyList) { return; @@ -49,7 +60,7 @@ function nextProxy(store: Store) { } logger.debug( - `ℹ [${store.name}] Next proxy index: ${store.currentProxyIndex} / Count: ${ + `ℹ [${store.name}] Next proxy index: ${store.currentProxyIndex} / Count: ${ store.proxyList.length } (${store.proxyList[store.currentProxyIndex]})` ); @@ -147,10 +158,129 @@ async function handleAdBlock(request: HTTPRequest, adBlockRequestHandler: any) { }); } +async function processLink(browser: Browser, store: Store, link: Link) { + if (config.page.inStockWaitTime && inStock[link.url]) { + logger.info(Print.inStockWaiting(link, store, true)); + return; + } + + const proxy = nextProxy(store); + const useAdBlock = !config.browser.lowBandwidth && !store.disableAdBlocker; + const customContext = config.browser.isIncognito; + + const context = customContext + ? await browser.createIncognitoBrowserContext() + : browser.defaultBrowserContext(); + const page = await context.newPage(); + await page.setRequestInterception(true); + + page.setDefaultNavigationTimeout(config.page.timeout); + await page.setUserAgent(await getRandomUserAgent()); + + let adBlockRequestHandler: any; + let pageProxy; + if (useAdBlock) { + const onProxyFunc = (event: keyof PageEventObject, handler: any) => { + if (event !== 'request') { + page.on(event, handler); + return; + } + + adBlockRequestHandler = handler; + }; + + pageProxy = new Proxy(page, { + get(target, prop, receiver) { + if (prop === 'on') { + return onProxyFunc; + } + + if (prop === 'setRequestInterception') { + return noop; + } + + return Reflect.get(target, prop, receiver); + }, + }); + await enableBlockerInPage(pageProxy); + } + + await page.setRequestInterception(true); + page.on('request', async request => { + if (await handleLowBandwidth(request)) { + return; + } + + if (await handleAdBlock(request, adBlockRequestHandler)) { + return; + } + + if (await handleProxy(request, proxy)) { + return; + } + + try { + await request.continue(); + } catch { + logger.debug('Failed to continue request.'); + } + }); + + if (store.captchaDeterrent) { + await runCaptchaDeterrent(browser, store, page); + } + + let statusCode = 0; + + try { + markStatusChecking(store.name, link); + statusCode = await lookupIem(browser, store, page, link); + } catch (error: unknown) { + markStatusResult(store.name, link, 'error'); + if (store.currentProxyIndex !== undefined && store.proxyList) { + const proxyLabel = `${store.currentProxyIndex + 1}/${store.proxyList.length}`; + logger.error( + `✖ [${proxyLabel}] [${store.name}] ${link.brand} ${link.series} ${ + link.model + } - ${(error as Error).message}` + ); + } else { + logger.error( + `✖ [${store.name}] ${link.brand} ${link.series} ${link.model} - ${ + (error as Error).message + }` + ); + } + const client = await page.target().createCDPSession(); + await client.send('Network.clearBrowserCookies'); + } + + if (pageProxy) { + await disableBlockerInPage(pageProxy); + } + + if ( + store.currentProxyIndex !== undefined && + store.proxyList && + store.proxyList.length > 1 + ) { + const client = await page.target().createCDPSession(); + await client.send('Network.clearBrowserCookies'); + } + + // Must apply backoff before closing the page, e.g. if CloudFlare is + // used to detect bot traffic, it introduces a 5 second page delay + // before redirecting to the next page + await processBackoffDelay(store, link, statusCode); + await closePage(page); + if (customContext) { + await context.close(); + } +} + /** - * Responsible for looking up information about a each product within - * a `Store`. It's important that we ignore `no-await-in-loop` here - * because we don't want to get rate limited within the same store. + * Responsible for looking up information about products within a `Store`. + * Concurrency is configurable via LOOKUP_THREADS and defaults to 1. * * @param browser Puppeteer browser. * @param store Vendor of items. @@ -174,133 +304,41 @@ async function lookup(browser: Browser, store: Store) { } } - /* eslint-disable no-await-in-loop */ - for (const link of store.links) { - if (!filterStoreLink(link)) { - continue; - } + const baseLinks = store.links.filter(link => filterStoreLink(link)); + const links = config.page.randomizeLookupOrder + ? shuffleArray(baseLinks) + : baseLinks; + const concurrency = Math.max(1, Math.floor(config.page.lookupThreads || 1)); - if (config.page.inStockWaitTime && inStock[link.url]) { - logger.info(Print.inStockWaiting(link, store, true)); - continue; + if (concurrency === 1 || links.length <= 1) { + /* eslint-disable no-await-in-loop */ + for (const link of links) { + await processLink(browser, store, link); } + /* eslint-enable no-await-in-loop */ + return; + } - const proxy = nextProxy(store); - - const useAdBlock = !config.browser.lowBandwidth && !store.disableAdBlocker; - const customContext = config.browser.isIncognito; - - const context = customContext - ? await browser.createIncognitoBrowserContext() - : browser.defaultBrowserContext(); - const page = await context.newPage(); - await page.setRequestInterception(true); - - page.setDefaultNavigationTimeout(config.page.timeout); - await page.setUserAgent(await getRandomUserAgent()); + logger.debug( + `[${store.name}] running ${links.length} links with concurrency ${concurrency}` + ); - let adBlockRequestHandler: any; - let pageProxy; - if (useAdBlock) { - const onProxyFunc = (event: keyof PageEventObject, handler: any) => { - if (event !== 'request') { - page.on(event, handler); + let nextIndex = 0; + const workers = Array.from( + {length: Math.min(concurrency, links.length)}, + async () => { + while (true) { + const currentIndex = nextIndex++; + if (currentIndex >= links.length) { return; } - adBlockRequestHandler = handler; - }; - - pageProxy = new Proxy(page, { - get(target, prop, receiver) { - if (prop === 'on') { - return onProxyFunc; - } - - // Give dummy setRequestInterception to avoid AdBlock from messing with it - if (prop === 'setRequestInterception') { - return noop; - } - - return Reflect.get(target, prop, receiver); - }, - }); - await enableBlockerInPage(pageProxy); - } - - await page.setRequestInterception(true); - page.on('request', async request => { - if (await handleLowBandwidth(request)) { - return; - } - - if (await handleAdBlock(request, adBlockRequestHandler)) { - return; - } - - if (await handleProxy(request, proxy)) { - return; + await processLink(browser, store, links[currentIndex]); } - - try { - await request.continue(); - } catch { - logger.debug('Failed to continue request.'); - } - }); - - if (store.captchaDeterrent) { - await runCaptchaDeterrent(browser, store, page); - } - - let statusCode = 0; - - try { - statusCode = await lookupIem(browser, store, page, link); - } catch (error: unknown) { - if (store.currentProxyIndex !== undefined && store.proxyList) { - const proxy = `${store.currentProxyIndex + 1}/${ - store.proxyList.length - }`; - logger.error( - `✖ [${proxy}] [${store.name}] ${link.brand} ${link.series} ${ - link.model - } - ${(error as Error).message}` - ); - } else { - logger.error( - `✖ [${store.name}] ${link.brand} ${link.series} ${link.model} - ${ - (error as Error).message - }` - ); - } - const client = await page.target().createCDPSession(); - await client.send('Network.clearBrowserCookies'); - } - - if (pageProxy) { - await disableBlockerInPage(pageProxy); - } - - if ( - store.currentProxyIndex !== undefined && - store.proxyList && - store.proxyList?.length > 1 - ) { - const client = await page.target().createCDPSession(); - await client.send('Network.clearBrowserCookies'); } + ); - // Must apply backoff before closing the page, e.g. if CloudFlare is - // used to detect bot traffic, it introduces a 5 second page delay - // before redirecting to the next page - await processBackoffDelay(store, link, statusCode); - await closePage(page); - if (customContext) { - await context.close(); - } - } - /* eslint-enable no-await-in-loop */ + await Promise.all(workers); } async function lookupIem( @@ -318,10 +356,12 @@ async function lookupIem( const statusCode = await handleResponse(browser, store, page, link, response); if (!isStatusCodeInRange(statusCode, successStatusCodes)) { + markStatusResult(store.name, link, 'error'); return statusCode; } if (await isItemInStock(store, page, link)) { + markStatusResult(store.name, link, 'in_stock'); const givenUrl = link.cartUrl && config.store.autoAddToCart ? link.cartUrl : link.url; logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`); @@ -343,7 +383,7 @@ async function lookupIem( } if (config.page.screenshot) { - logger.debug('ℹ saving screenshot'); + logger.debug('ℹ saving screenshot'); await fs.mkdir(config.page.screenshotDir, {recursive: true}); link.screenshot = path.join( @@ -352,8 +392,12 @@ async function lookupIem( ); await page.screenshot({path: link.screenshot}); } + + return statusCode; } + markStatusResult(store.name, link, 'out_of_stock'); + return statusCode; } @@ -558,7 +602,7 @@ async function runCaptchaDeterrent(browser: Browser, store: Store, page: Page) { if (!isStatusCodeInRange(statusCode, successStatusCodes)) { logger.warn( - `✖ [${store.name}] - Failed to navigate to anti-captcha target: ${link.url}` + `✖ [${store.name}] - Failed to navigate to anti-captcha target: ${link.url}` ); } } diff --git a/src/web/index.ts b/src/web/index.ts index beb1ef7246..5f18f740a3 100644 --- a/src/web/index.ts +++ b/src/web/index.ts @@ -11,6 +11,13 @@ import { storeList, updateStores, } from '../store/model'; +import {getMatrixStatus, initializeStatusMap} from './status'; +import { + readSettingsFile, + writeSettingsRaw, + writeSettingsValues, +} from './settings'; +import {restartBot} from '../runtime-control'; const approot = join(__dirname, '../../../'); const webroot = join(approot, './web'); @@ -112,6 +119,7 @@ function handleAPI( try { setConfig(JSON.parse(data.join(''))); updateStores(); + initializeStatusMap(); } catch { logger.warn('Could not setup config'); } @@ -138,6 +146,68 @@ function handleAPI( case 'models': sendJSON(response, getAllModels()); return; + case 'matrix': + sendJSON(response, getMatrixStatus()); + return; + case 'control': + if (urlComponents[2] === 'restart' && request.method === 'POST') { + restartBot() + .then(() => sendJSON(response, {ok: true})) + .catch((error: unknown) => { + logger.warn(`Could not restart bot: ${(error as Error).message}`); + sendError(response, 'Could not restart bot', 500); + }); + return; + } + + sendError(response, 'No control route found for path', 400); + return; + case 'settings': + if (request.method === 'PUT') { + const data: string[] = []; + request.on('data', chunk => { + data.push(chunk); + }); + request.on('end', () => { + try { + const payload = JSON.parse(data.join('')); + const result = + typeof payload.raw === 'string' + ? writeSettingsRaw(payload.raw) + : writeSettingsValues(payload.values ?? {}); + + if (result.values.LOOKUP_THREADS !== undefined) { + setConfig({ + page: { + ...config.page, + lookupThreads: Number(result.values.LOOKUP_THREADS) || 1, + }, + }); + } + + if (result.values.RANDOMIZE_LOOKUP_ORDER !== undefined) { + setConfig({ + page: { + ...config.page, + randomizeLookupOrder: + result.values.RANDOMIZE_LOOKUP_ORDER === 'true', + }, + }); + } + + sendJSON(response, result); + return; + } catch (error: unknown) { + logger.warn(`Could not update settings: ${(error as Error).message}`); + sendError(response, 'Could not update settings', 400); + return; + } + }); + return; + } + + sendJSON(response, readSettingsFile()); + return; case 'screenshots': if (urlComponents.length >= 3) { const timeStamp = urlComponents[2]; @@ -193,6 +263,7 @@ let server: Server | undefined; export async function startAPIServer() { await stopAPIServer(); if (process.env.WEB_PORT) { + initializeStatusMap(); server = createServer(requestListener); server.listen(Number(process.env.WEB_PORT)); } diff --git a/src/web/settings.ts b/src/web/settings.ts new file mode 100644 index 0000000000..ac36c77748 --- /dev/null +++ b/src/web/settings.ts @@ -0,0 +1,119 @@ +import dotenv from 'dotenv'; +import {existsSync, readFileSync, writeFileSync} from 'fs'; +import {activeConfigPath} from '../config'; + +type EnvMap = Record; + +function normalizeValue(value: unknown) { + if (value === undefined || value === null) { + return ''; + } + + return String(value); +} + +function parseEnvFileContents(contents: string): EnvMap { + const parsed = dotenv.parse(contents); + const result: EnvMap = {}; + + for (const [key, value] of Object.entries(parsed)) { + result[key] = value; + } + + return result; +} + +function serializeEnvValue(value: string) { + if (value === '') { + return ''; + } + + if (value.includes('\n')) { + return JSON.stringify(value); + } + + if (/\s|#|"/.test(value)) { + return JSON.stringify(value); + } + + return value; +} + +export function getSettingsFilePath() { + return activeConfigPath; +} + +export function readSettingsFile() { + const filePath = getSettingsFilePath(); + const contents = existsSync(filePath) ? readFileSync(filePath, 'utf8') : ''; + + return { + filePath, + raw: contents, + values: parseEnvFileContents(contents), + }; +} + +export function writeSettingsValues(updatedValues: Record) { + const {filePath, raw, values} = readSettingsFile(); + const nextValues: EnvMap = {...values}; + + for (const [key, value] of Object.entries(updatedValues)) { + nextValues[key] = normalizeValue(value); + process.env[key] = nextValues[key]; + } + + const lines = raw.length > 0 ? raw.split(/\r?\n/) : []; + const seenKeys = new Set(); + + const rewritten = lines.map(line => { + const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) { + return line; + } + + const key = match[1]; + if (!(key in nextValues)) { + return line; + } + + seenKeys.add(key); + return `${key}=${serializeEnvValue(nextValues[key])}`; + }); + + for (const [key, value] of Object.entries(nextValues)) { + if (seenKeys.has(key)) { + continue; + } + + rewritten.push(`${key}=${serializeEnvValue(value)}`); + } + + const finalText = rewritten.join('\n').replace(/\n?$/, '\n'); + writeFileSync(filePath, finalText, 'utf8'); + + return { + filePath, + raw: finalText, + values: nextValues, + }; +} + +export function writeSettingsRaw(raw: string) { + const filePath = getSettingsFilePath(); + writeFileSync(filePath, raw.replace(/\r\n/g, '\n').replace(/\n?$/, '\n')); + + const contents = readFileSync(filePath, 'utf8'); + const values = parseEnvFileContents(contents); + + for (const [key, value] of Object.entries(values)) { + process.env[key] = value; + } + + return { + filePath, + raw: contents, + values, + }; +} + diff --git a/src/web/status.ts b/src/web/status.ts new file mode 100644 index 0000000000..3718596cd3 --- /dev/null +++ b/src/web/status.ts @@ -0,0 +1,169 @@ +import {config} from '../config'; +import {filterStoreLink} from '../store/filter'; +import {Link, Series, getStores} from '../store/model'; + +export type MatrixCellStatus = + | 'idle' + | 'checking' + | 'out_of_stock' + | 'in_stock' + | 'error' + | 'unsupported'; + +type StatusEntry = { + lastCheckedAt?: number; + lastUpdatedAt?: number; + productCount: number; + status: MatrixCellStatus; + url?: string; +}; + +const statusMap = new Map(); + +function getKey(storeName: string, series: Series) { + return `${storeName}::${series}`; +} + +function getSelectedStoreEntries() { + return [...getStores().entries()]; +} + +function getFilteredLinksForStore(storeName: string) { + const store = getStores().get(storeName); + if (!store) { + return [] as Link[]; + } + + return store.links.filter((link: Link) => filterStoreLink(link)); +} + +export function getSelectedSeriesMatrix() { + if (config.store.showOnlySeries.length > 0) { + return [...config.store.showOnlySeries].sort() as Series[]; + } + + const selected = new Set(); + + for (const [storeName] of getSelectedStoreEntries()) { + for (const link of getFilteredLinksForStore(storeName)) { + selected.add(link.series); + } + } + + return [...selected].sort() as Series[]; +} + +export function initializeStatusMap() { + const selectedSeries = getSelectedSeriesMatrix(); + const selectedStores = getSelectedStoreEntries().map(([storeName]) => storeName); + + for (const storeName of selectedStores) { + const links = getFilteredLinksForStore(storeName); + + for (const series of selectedSeries) { + const key = getKey(storeName, series); + const supportedLinks = links.filter((link: Link) => link.series === series); + const existing = statusMap.get(key); + + statusMap.set(key, { + lastCheckedAt: existing?.lastCheckedAt, + lastUpdatedAt: existing?.lastUpdatedAt, + productCount: supportedLinks.length, + status: existing?.status ?? 'idle', + url: + existing?.url ?? + supportedLinks.find((link: Link) => link.cartUrl)?.cartUrl ?? + supportedLinks[0]?.url, + }); + } + } + + for (const key of [...statusMap.keys()]) { + const [storeName, series] = key.split('::'); + if ( + !selectedStores.includes(storeName) || + !selectedSeries.includes(series as Series) + ) { + statusMap.delete(key); + } + } +} + +function ensureEntry(storeName: string, link: Link): StatusEntry { + const key = getKey(storeName, link.series); + const existing = statusMap.get(key); + if (existing) { + if (!existing.url) { + existing.url = link.cartUrl ?? link.url; + } + return existing; + } + + const created: StatusEntry = { + lastCheckedAt: undefined, + lastUpdatedAt: undefined, + productCount: 1, + status: 'idle', + url: link.cartUrl ?? link.url, + }; + statusMap.set(key, created); + return created; +} + +export function markStatusChecking(storeName: string, link: Link) { + const entry = ensureEntry(storeName, link); + entry.lastCheckedAt = Date.now(); + entry.lastUpdatedAt = Date.now(); + entry.status = 'checking'; +} + +export function markStatusResult( + storeName: string, + link: Link, + status: Extract +) { + const entry = ensureEntry(storeName, link); + entry.lastCheckedAt = Date.now(); + entry.lastUpdatedAt = Date.now(); + entry.url = link.cartUrl ?? link.url; + entry.status = status; +} + +export function getMatrixStatus() { + initializeStatusMap(); + + const selectedStores = getSelectedStoreEntries().map(([storeName]) => storeName); + const selectedSeries = getSelectedSeriesMatrix(); + + return { + generatedAt: Date.now(), + stores: selectedStores, + series: selectedSeries, + cells: selectedSeries.map(series => { + return { + series, + stores: selectedStores.map(storeName => { + const entry = statusMap.get(getKey(storeName, series)) ?? { + productCount: 0, + status: 'unsupported' as MatrixCellStatus, + }; + + return { + lastCheckedAt: entry.lastCheckedAt ?? null, + lastUpdatedAt: entry.lastUpdatedAt ?? null, + productCount: entry.productCount, + status: entry.status, + store: storeName, + url: entry.url ?? null, + }; + }), + }; + }), + summary: { + selectedSeries: selectedSeries.length, + selectedStores: selectedStores.length, + webPortEnabled: Boolean(process.env.WEB_PORT), + openBrowser: config.browser.open, + }, + }; +} diff --git a/web/index.html b/web/index.html index 6c7876f179..c35047157e 100644 --- a/web/index.html +++ b/web/index.html @@ -1,189 +1,1117 @@ - - - streetmerchant control - - - - - - - - - - - - - - - - - -
StoresBrandsSeriesModels
- -
- - -



- Screenshots (Refresh) -
- + + + + + streetmerchant + + + + +
+
+
+

streetmerchant

+
+
+
+ +
+
+ +
+
+ +
+
+
+ Stores + All selected +
+ +
+
+ + + +
+
+
+ + +
+
+
+
+ +
+
+
+ Series + All selected +
+ +
+
+ + + +
+
+
+ + +
+
+
+
+ +
+
+
+ Models + All selected +
+ +
+
+ + + +
+
+
+ + +
+
+
+
0 stores
+
0 series
+
Updated --:--:--
+
+
+ +
+
+
checking
+
out of stock
+
in stock
+
idle or error
+
unsupported
+
+ +
+
+ +
+
Cells open the latest product or cart URL when one is available.
+
Refresh cadence: every second.
+
+
+
+
+
+ Environment Settings + Edit the active dotenv file used by the running app. +
+
+ +
+
File changes are persisted. Some settings may need a restart to fully take effect.
+
+ + +
+
+
+
+ From a895a0ccf04158887c720a08757d6aa53162fedd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 00:00:39 +0000 Subject: [PATCH 06/13] Initial plan From 734e3ef82e7c1f7724ef2b7c280da916ed4cbbb4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 00:17:05 +0000 Subject: [PATCH 07/13] Fix lint errors: Prettier formatting and no-constant-condition Agent-Logs-Url: https://github.com/Mr-Tech-13/streetmerchant/sessions/822a04fb-2dca-44b3-83f4-f369fcf214ba Co-authored-by: Mr-Tech-13 <81822275+Mr-Tech-13@users.noreply.github.com> --- src/runtime-control.ts | 1 - src/store/lookup.ts | 18 +++++++++--------- src/store/model/nvidia-de.ts | 24 ++++++++++++++++++++++-- src/store/model/nvidia-fr.ts | 26 +++++++++++++++++++++++--- src/store/model/nvidia-gb.ts | 35 +++++++++++++++++++++++++++++++---- src/web/index.ts | 4 +++- src/web/settings.ts | 1 - src/web/status.ts | 12 +++++++++--- 8 files changed, 97 insertions(+), 24 deletions(-) diff --git a/src/runtime-control.ts b/src/runtime-control.ts index d790992488..48b009769a 100644 --- a/src/runtime-control.ts +++ b/src/runtime-control.ts @@ -11,4 +11,3 @@ export async function restartBot() { await restartBotHandler(); } - diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 1448a3b789..c8f3af4a86 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -60,9 +60,11 @@ function nextProxy(store: Store) { } logger.debug( - `ℹ [${store.name}] Next proxy index: ${store.currentProxyIndex} / Count: ${ - store.proxyList.length - } (${store.proxyList[store.currentProxyIndex]})` + `ℹ [${store.name}] Next proxy index: ${ + store.currentProxyIndex + } / Count: ${store.proxyList.length} (${ + store.proxyList[store.currentProxyIndex] + })` ); return store.proxyList[store.currentProxyIndex]; @@ -238,7 +240,9 @@ async function processLink(browser: Browser, store: Store, link: Link) { } catch (error: unknown) { markStatusResult(store.name, link, 'error'); if (store.currentProxyIndex !== undefined && store.proxyList) { - const proxyLabel = `${store.currentProxyIndex + 1}/${store.proxyList.length}`; + const proxyLabel = `${store.currentProxyIndex + 1}/${ + store.proxyList.length + }`; logger.error( `✖ [${proxyLabel}] [${store.name}] ${link.brand} ${link.series} ${ link.model @@ -327,12 +331,8 @@ async function lookup(browser: Browser, store: Store) { const workers = Array.from( {length: Math.min(concurrency, links.length)}, async () => { - while (true) { + while (nextIndex < links.length) { const currentIndex = nextIndex++; - if (currentIndex >= links.length) { - return; - } - await processLink(browser, store, links[currentIndex]); } } diff --git a/src/store/model/nvidia-de.ts b/src/store/model/nvidia-de.ts index e8de308d1c..7536ff068b 100644 --- a/src/store/model/nvidia-de.ts +++ b/src/store/model/nvidia-de.ts @@ -4,11 +4,11 @@ export const NvidiaDE: Store = { currency: '€', labels: { inStock: { - container: '.buy', + container: 'span.buy-link-atc', text: ['In den Einkaufswagen', 'JETZT KAUFEN'], }, outOfStock: { - container: '.buy', + container: 'span.buy-link-atc', text: ['DERZEIT NICHT VERFÜGBAR'], }, }, @@ -55,7 +55,27 @@ export const NvidiaDE: Store = { series: '3090', url: 'https://shop.nvidia.com/de-de/geforce/store/gpu/?page=1&limit=9&locale=de-de&category=GPU&gpu=RTX%203090&manufacturer=NVIDIA', }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5080', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5090', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + }, ], name: 'nvidia-de', country: 'DE', }; +/* Copy Paste Template + { + brand: 'nvidia', + model: 'founders edition', + series: '', + url: '', + }, +*/ diff --git a/src/store/model/nvidia-fr.ts b/src/store/model/nvidia-fr.ts index b236c50446..d9cfdcccd0 100644 --- a/src/store/model/nvidia-fr.ts +++ b/src/store/model/nvidia-fr.ts @@ -4,11 +4,11 @@ export const NvidiaFR: Store = { currency: '€', labels: { inStock: { - container: '.buy', - text: ['ajouter au panier', 'acheter maintenant'], + container: 'span.buy-link-atc', + text: ['Acheter maintenant'], }, outOfStock: { - container: '.buy', + container: 'span.buy-link-atc', text: ['RUPTURE DE STOCK'], }, }, @@ -55,7 +55,27 @@ export const NvidiaFR: Store = { series: '3090', url: 'https://shop.nvidia.com/fr-fr/geforce/store/gpu/?page=1&limit=9&locale=fr-fr&category=GPU&gpu=RTX%203090&manufacturer=NVIDIA', }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5090', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5080', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', + }, ], name: 'nvidia-fr', country: 'FR', }; +/* Copy Paste Template + { + brand: 'nvidia', + model: 'founders edition', + series: '', + url: '', + }, +*/ diff --git a/src/store/model/nvidia-gb.ts b/src/store/model/nvidia-gb.ts index 52a460939d..512f736959 100644 --- a/src/store/model/nvidia-gb.ts +++ b/src/store/model/nvidia-gb.ts @@ -4,12 +4,12 @@ export const NvidiaGB: Store = { currency: '£', labels: { inStock: { - container: '.buy', - text: ['add to cart', 'buy now'], + container: 'span.buy-link-atc', + text: ['Buy Now', 'Add to Cart'], }, outOfStock: { - container: '.buy', - text: ['out of stock'], + container: 'span.buy-link-atc', + text: ['Out of Stock'], }, }, links: [ @@ -55,7 +55,34 @@ export const NvidiaGB: Store = { series: '3090', url: 'https://shop.nvidia.com/en-gb/geforce/store/gpu/?page=1&limit=9&locale=en-gb&category=GPU&gpu=RTX%203090&manufacturer=NVIDIA', }, + { + brand: 'asus', + model: 'dual', + series: '4070', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&manufacturer=ASUS&gpu=RTX%204070&', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5090', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + }, + { + brand: 'nvidia', + model: 'founders edition', + series: '5080', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&manufacturer=NVIDIA&gpu=RTX%205080&', + }, ], name: 'nvidia-gb', country: 'UK', }; + +/* Copy Paste Template + { + brand: 'nvidia', + model: 'founders edition', + series: '', + url: '', + }, +*/ diff --git a/src/web/index.ts b/src/web/index.ts index 5f18f740a3..64e1278205 100644 --- a/src/web/index.ts +++ b/src/web/index.ts @@ -198,7 +198,9 @@ function handleAPI( sendJSON(response, result); return; } catch (error: unknown) { - logger.warn(`Could not update settings: ${(error as Error).message}`); + logger.warn( + `Could not update settings: ${(error as Error).message}` + ); sendError(response, 'Could not update settings', 400); return; } diff --git a/src/web/settings.ts b/src/web/settings.ts index ac36c77748..6632d81fdf 100644 --- a/src/web/settings.ts +++ b/src/web/settings.ts @@ -116,4 +116,3 @@ export function writeSettingsRaw(raw: string) { values, }; } - diff --git a/src/web/status.ts b/src/web/status.ts index 3718596cd3..65b5afa834 100644 --- a/src/web/status.ts +++ b/src/web/status.ts @@ -55,14 +55,18 @@ export function getSelectedSeriesMatrix() { export function initializeStatusMap() { const selectedSeries = getSelectedSeriesMatrix(); - const selectedStores = getSelectedStoreEntries().map(([storeName]) => storeName); + const selectedStores = getSelectedStoreEntries().map( + ([storeName]) => storeName + ); for (const storeName of selectedStores) { const links = getFilteredLinksForStore(storeName); for (const series of selectedSeries) { const key = getKey(storeName, series); - const supportedLinks = links.filter((link: Link) => link.series === series); + const supportedLinks = links.filter( + (link: Link) => link.series === series + ); const existing = statusMap.get(key); statusMap.set(key, { @@ -132,7 +136,9 @@ export function markStatusResult( export function getMatrixStatus() { initializeStatusMap(); - const selectedStores = getSelectedStoreEntries().map(([storeName]) => storeName); + const selectedStores = getSelectedStoreEntries().map( + ([storeName]) => storeName + ); const selectedSeries = getSelectedSeriesMatrix(); return { From 355f47e31d50563742230142afacdd6c7a845ca1 Mon Sep 17 00:00:00 2001 From: agpuperson <81822275+Mr-Tech-13@users.noreply.github.com> Date: Thu, 26 Mar 2026 23:04:34 -0400 Subject: [PATCH 08/13] Fix duplicate URL entry for NVIDIA graphics cards --- src/store/model/nvidia-de.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/store/model/nvidia-de.ts b/src/store/model/nvidia-de.ts index b31d8ee3bd..bb919e8407 100644 --- a/src/store/model/nvidia-de.ts +++ b/src/store/model/nvidia-de.ts @@ -59,14 +59,12 @@ export const NvidiaDE: Store = { brand: 'nvidia', model: 'founders edition', series: '5080', - url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', - url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', }, { brand: 'nvidia', model: 'founders edition', series: '5090', - url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', }, ], From c6710c482f7d6789b4e4f70f0e085a59d39d9b86 Mon Sep 17 00:00:00 2001 From: agpuperson Date: Wed, 1 Apr 2026 19:34:22 -0400 Subject: [PATCH 09/13] Fix Linting --- src/store/lookup.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/store/lookup.ts b/src/store/lookup.ts index c8f3af4a86..bc456627df 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -60,7 +60,7 @@ function nextProxy(store: Store) { } logger.debug( - `ℹ [${store.name}] Next proxy index: ${ + `[INFO] [${store.name}] Next proxy index: ${ store.currentProxyIndex } / Count: ${store.proxyList.length} (${ store.proxyList[store.currentProxyIndex] @@ -244,13 +244,13 @@ async function processLink(browser: Browser, store: Store, link: Link) { store.proxyList.length }`; logger.error( - `✖ [${proxyLabel}] [${store.name}] ${link.brand} ${link.series} ${ + `[ERROR] [${proxyLabel}] [${store.name}] ${link.brand} ${link.series} ${ link.model } - ${(error as Error).message}` ); } else { logger.error( - `✖ [${store.name}] ${link.brand} ${link.series} ${link.model} - ${ + `[ERROR] [${store.name}] ${link.brand} ${link.series} ${link.model} - ${ (error as Error).message }` ); @@ -383,7 +383,7 @@ async function lookupIem( } if (config.page.screenshot) { - logger.debug('ℹ saving screenshot'); + logger.debug('[INFO] saving screenshot'); await fs.mkdir(config.page.screenshotDir, {recursive: true}); link.screenshot = path.join( @@ -602,7 +602,7 @@ async function runCaptchaDeterrent(browser: Browser, store: Store, page: Page) { if (!isStatusCodeInRange(statusCode, successStatusCodes)) { logger.warn( - `✖ [${store.name}] - Failed to navigate to anti-captcha target: ${link.url}` + `[ERROR] [${store.name}] - Failed to navigate to anti-captcha target: ${link.url}` ); } } From d683635f70e0c60d6a2184d6ddba6254a1ef1857 Mon Sep 17 00:00:00 2001 From: agpuperson Date: Wed, 1 Apr 2026 19:35:57 -0400 Subject: [PATCH 10/13] Fix Linting (again) --- src/store/model/nvidia-de.ts | 12 ++++++------ src/store/model/nvidia-fr.ts | 8 ++++---- src/store/model/nvidia-gb.ts | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/store/model/nvidia-de.ts b/src/store/model/nvidia-de.ts index bb919e8407..2ed914e048 100644 --- a/src/store/model/nvidia-de.ts +++ b/src/store/model/nvidia-de.ts @@ -1,7 +1,7 @@ import {Store} from './store'; export const NvidiaDE: Store = { - currency: '€', + currency: '€', labels: { inStock: { container: 'span.buy-link-atc', @@ -9,8 +9,8 @@ export const NvidiaDE: Store = { }, outOfStock: { container: 'span.buy-link-atc', - text: ['DERZEIT NICHT VERFÜGBAR'], - }, + text: ['DERZEIT NICHT VERFÃœGBAR'], + }, }, links: [ { @@ -59,13 +59,13 @@ export const NvidiaDE: Store = { brand: 'nvidia', model: 'founders edition', series: '5080', - url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', }, { brand: 'nvidia', model: 'founders edition', series: '5090', - url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + url: 'https://marketplace.nvidia.com/de-de/consumer/graphics-cards/?locale=de-de&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', }, ], name: 'nvidia-de', @@ -76,6 +76,6 @@ export const NvidiaDE: Store = { brand: 'nvidia', model: 'founders edition', series: '', - url: '', + url: '', }, */ diff --git a/src/store/model/nvidia-fr.ts b/src/store/model/nvidia-fr.ts index 99575f38a6..790dcea95a 100644 --- a/src/store/model/nvidia-fr.ts +++ b/src/store/model/nvidia-fr.ts @@ -1,7 +1,7 @@ import {Store} from './store'; export const NvidiaFR: Store = { - currency: '€', + currency: '€', labels: { inStock: { container: 'span.buy-link-atc', @@ -59,13 +59,13 @@ export const NvidiaFR: Store = { brand: 'nvidia', model: 'founders edition', series: '5090', - url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', }, { brand: 'nvidia', model: 'founders edition', series: '5080', - url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', + url: 'https://marketplace.nvidia.com/fr-fr/consumer/graphics-cards/?locale=fr-fr&page=1&limit=12&gpu=RTX%205080&manufacturer=NVIDIA', }, ], name: 'nvidia-fr', @@ -76,6 +76,6 @@ export const NvidiaFR: Store = { brand: 'nvidia', model: 'founders edition', series: '', - url: '', + url: '', }, */ diff --git a/src/store/model/nvidia-gb.ts b/src/store/model/nvidia-gb.ts index 4101263570..0589f1f628 100644 --- a/src/store/model/nvidia-gb.ts +++ b/src/store/model/nvidia-gb.ts @@ -1,11 +1,11 @@ import {Store} from './store'; export const NvidiaGB: Store = { - currency: '£', + currency: '£', labels: { inStock: { container: 'span.buy-link-atc', - text: ['Buy Now', 'Add to Cart'] + text: ['Buy Now', 'Add to Cart'], }, outOfStock: { container: 'span.buy-link-atc', @@ -65,13 +65,13 @@ export const NvidiaGB: Store = { brand: 'nvidia', model: 'founders edition', series: '5090', - url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&gpu=RTX%205090&manufacturer=NVIDIA', }, { brand: 'nvidia', model: 'founders edition', series: '5080', - url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&manufacturer=NVIDIA&gpu=RTX%205080&', + url: 'https://marketplace.nvidia.com/en-gb/consumer/graphics-cards/?locale=en-gb&page=1&limit=12&manufacturer=NVIDIA&gpu=RTX%205080&', }, ], name: 'nvidia-gb', @@ -83,6 +83,6 @@ export const NvidiaGB: Store = { brand: 'nvidia', model: 'founders edition', series: '', - url: '', + url: '', }, */ From f52ee44a10c9e3595a75c610871864195d503979 Mon Sep 17 00:00:00 2001 From: agpuperson Date: Wed, 1 Apr 2026 19:38:33 -0400 Subject: [PATCH 11/13] Fix compiling errors --- src/store/model/nvidia-de.ts | 4 ++-- src/store/model/nvidia-fr.ts | 2 +- src/store/model/nvidia-gb.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/store/model/nvidia-de.ts b/src/store/model/nvidia-de.ts index 2ed914e048..09df63bbfc 100644 --- a/src/store/model/nvidia-de.ts +++ b/src/store/model/nvidia-de.ts @@ -1,7 +1,7 @@ import {Store} from './store'; export const NvidiaDE: Store = { - currency: '€', + currency: '€', labels: { inStock: { container: 'span.buy-link-atc', @@ -9,7 +9,7 @@ export const NvidiaDE: Store = { }, outOfStock: { container: 'span.buy-link-atc', - text: ['DERZEIT NICHT VERFÃœGBAR'], + text: ['DERZEIT NICHT VERFÃœGBAR'], }, }, links: [ diff --git a/src/store/model/nvidia-fr.ts b/src/store/model/nvidia-fr.ts index 790dcea95a..d9cfdcccd0 100644 --- a/src/store/model/nvidia-fr.ts +++ b/src/store/model/nvidia-fr.ts @@ -1,7 +1,7 @@ import {Store} from './store'; export const NvidiaFR: Store = { - currency: '€', + currency: '€', labels: { inStock: { container: 'span.buy-link-atc', diff --git a/src/store/model/nvidia-gb.ts b/src/store/model/nvidia-gb.ts index 0589f1f628..512f736959 100644 --- a/src/store/model/nvidia-gb.ts +++ b/src/store/model/nvidia-gb.ts @@ -1,7 +1,7 @@ import {Store} from './store'; export const NvidiaGB: Store = { - currency: '£', + currency: '£', labels: { inStock: { container: 'span.buy-link-atc', From f799e51151e5ab89cb9f785465afae07f9a1d601 Mon Sep 17 00:00:00 2001 From: agpuperson <81822275+Mr-Tech-13@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:49:54 -0400 Subject: [PATCH 12/13] fix dotenv-example --- dotenv-example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dotenv-example b/dotenv-example index e5433e0d42..3b2c179dbd 100644 --- a/dotenv-example +++ b/dotenv-example @@ -74,8 +74,8 @@ IN_STOCK_WAIT_TIME= INCOGNITO= LOG_LEVEL= LOW_BANDWIDTH= -LOOKUP_THREADS=1 -RANDOMIZE_LOOKUP_ORDER=true +LOOKUP_THREADS= +RANDOMIZE_LOOKUP_ORDER= MAX_PRICE_SERIES_3060= MAX_PRICE_SERIES_3060TI= MAX_PRICE_SERIES_3070= From bf1c99ec5058e1d4bb7cde3b1d95e9cceb9a75dd Mon Sep 17 00:00:00 2001 From: agpuperson Date: Wed, 1 Apr 2026 19:54:33 -0400 Subject: [PATCH 13/13] Update Documentation --- README.md | 3 +++ docs/getting-started.md | 20 ++++++++++++++++++++ docs/index.md | 1 + docs/reference/application.md | 5 ++++- docs/reference/filter.md | 5 ++++- 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d053ce0a7e..8fddea7f87 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ First and foremost, this service _will not_ automatically buy for you. - **Checks stock continuously** -- runs 24/7, 365, looking for the items you want. - **Ready for checkout** -- ability to add to cart when available and even opens the browser for you. +- **Live dashboard** -- optional web interface with a matrix view of selected stores and series, filter controls, dotenv editing, and restart control. - **Notifications galore** -- when you're not by your computer, worry free with notifications to most platforms and devices when an item comes in stock. ## Quick start @@ -25,4 +26,6 @@ git clone https://github.com/jef/streetmerchant.git cd streetmerchant && npm i && npm run start ``` +To enable the web dashboard, set `WEB_PORT` in `dotenv` and open `http://localhost:` while streetmerchant is running. + For more information and customization, visit [jef.buzz/streetmerchant/getting-started](https://jef.buzz/streetmerchant/getting-started). diff --git a/docs/getting-started.md b/docs/getting-started.md index 6fbc1fa531..8bb3d5a313 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -25,6 +25,26 @@ You do not need any computer skills, smarts, or anything of that nature. You are At any point you want the program to stop, use ++ctrl+c++. +## Using the web dashboard + +If you want to manage streetmerchant from the browser while it is running, set `WEB_PORT` in your `dotenv` file. + +```shell +WEB_PORT=8080 +``` + +After you start the app, open `http://localhost:8080`. + +The dashboard includes: + +- A live matrix with selected stores across the top and selected series on the left. +- Store, series, and model menus that update the running configuration. +- A settings editor for the active `dotenv` file. +- A restart button to reload the bot without closing the dashboard. + +???+ note + Filter changes made in the dashboard are persisted back to the active `dotenv` file. Some settings still require a restart before they fully take effect. + ???+ tip Community based help can also be found on the [wiki](https://github.com/jef/streetmerchant/wiki). Feel free to check that out if you're having problems running. If you're still having problems running, you're probably not the first. Make some searches through the [GitHub issues](https://github.com/jef/streetmerchant/issues) before making one. diff --git a/docs/index.md b/docs/index.md index 1cd1ae933a..dc61bd8002 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,6 +6,7 @@ First and foremost, this service _will not_ automatically buy for you. - **Checks stock continuously** -- runs 24/7, 365, looking for the items you want. - **Ready for checkout** -- ability to add to cart when available and even opens the browser for you. +- **Live dashboard** -- optional web interface with a matrix view of selected stores and series, filter controls, dotenv editing, and restart control. - **Notifications galore** -- when you're not by your computer, worry free with notifications to most platforms and devices when an item comes in stock. ## Getting started diff --git a/docs/reference/application.md b/docs/reference/application.md index 981b9f5de2..d4517579ee 100644 --- a/docs/reference/application.md +++ b/docs/reference/application.md @@ -7,6 +7,7 @@ | `HEADLESS` | Puppeteer to run headless or not. Debugging related, default: `true` | | `INCOGNITO` | Puppeteer to run incognito or not. Debugging related, default: `false` | | `IN_STOCK_WAIT_TIME` | Time to wait between requests to the same link if it has that card in stock. In seconds, default: `0` | +| `LOOKUP_THREADS` | Maximum number of product links checked concurrently per store loop, default: `1` | | `LOG_LEVEL` | [Logging levels](https://github.com/winstonjs/winston#logging-levels). Debugging related, default: `info` | | `LOW_BANDWIDTH` | Blocks images/fonts to reduce traffic. Disables ad blocker, default: `false` | | `NVIDIA_ADD_TO_CART_ATTEMPTS` | Maximum number of attempts add an item to card in the Nvidia storefront, default: `10` | @@ -20,11 +21,12 @@ | `PROXY_PROTOCOL` | Protocol of proxy server, such as `socks5`, default: `http` | | `PROXY_ADDRESS` | IP Address or fqdn of proxy server | | `PROXY_PORT` | TCP Port number on which the proxy is listening for connections, default: `80` | +| `RANDOMIZE_LOOKUP_ORDER` | Shuffle store and product lookup order on each pass, default: `false` | | `RESTART_TIME` | Restarts chrome after defined milliseconds. `0` for never, default: `0` | | `SCREENSHOT` | Capture screenshot of page if a card is found, default: `true` | | `SCREENSHOT_DIR` | The directory for saving the screenshots, default: `screenshots` | | `USER_AGENT` | Custom user agent used for requests | -| `WEB_PORT` | Starts a webserver to be able to control the bot while it is running. Setting this value starts this service. | +| `WEB_PORT` | Starts a webserver to control the bot while it is running. The dashboard includes a live matrix view, filter menus, dotenv editing, and a restart control. | ???+ info There is more information on proxy settings in the [Proxy documentation](proxy.md). @@ -32,3 +34,4 @@ ???+ tip - You can also have a list of proxies that are rotated while searching stores. Proxies can be read from a file named `STORENAME.proxies` in the format of `socks5://username:password@ip`; one per line. - Data usage is [known to be high](https://github.com/jef/streetmerchant/issues?q=is%3Aissue+sort%3Aupdated-desc+bandwidth). This is expected as the program scrapes many websites in parallel 24/7. To help reduce this, use `LOW_BANDWIDTH="true"`. We are looking into other solutions as well, but is low priority. + - Increasing `LOOKUP_THREADS` can improve throughput, but it also increases request pressure and rate-limit risk. diff --git a/docs/reference/filter.md b/docs/reference/filter.md index cbef82d22a..28c23bf38d 100644 --- a/docs/reference/filter.md +++ b/docs/reference/filter.md @@ -54,6 +54,9 @@ ???+ note For `MAX_PRICE_SERIES_*` variables: Use whole numbers only (no currency symbol is required). Avoid using any commas or decimal points. Example: `1234`. Merchandise found above this price will be skipped. +???+ info + When `WEB_PORT` is enabled, the web dashboard can update `STORES`, `SHOW_ONLY_SERIES`, and `SHOW_ONLY_MODELS` from the browser. Those changes are also written back to the active `dotenv` file. + ## Supported stores Used with the `STORES` variable. @@ -122,7 +125,7 @@ Used with the `STORES` variable. | Drako | IT | `drako` | | DustinHome | NO | `dustinhome-no` | | eBuyer | UK | `ebuyer` | -| El Corte Inglés | ES | `elcorteingles` | +| El Corte Ingles | ES | `elcorteingles` | | Eletronicamente | ES | `eletronicamente` | | Elkjop | NO | `elkjop` | | ePrice | IT | `eprice` |