From 022be9ced92e2a9416eaabd6b4f3541e79a26236 Mon Sep 17 00:00:00 2001 From: Matt Jonker Date: Mon, 1 Jun 2020 08:52:18 -0400 Subject: [PATCH 1/4] Uses URL to reliably generate the URL --- lib/captcha.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/captcha.js b/lib/captcha.js index 197496f..4d6a71c 100644 --- a/lib/captcha.js +++ b/lib/captcha.js @@ -35,11 +35,10 @@ const solve = async (response) => { body.push('h-captcha-response=' + captchaResponse); } - const baseUrl = response.url.substring(0, response.url.lastIndexOf('/')); const { method, action, enctype } = document.getElementById('challenge-form'); return { method, - url: action.startsWith('/') ? baseUrl + action : action, + url: new URL(action, response.url), headers: { 'content-type': enctype, // application/x-www-form-urlencoded }, From b5bea164aa2691af5a67d0bc9c78b9c6e7239f19 Mon Sep 17 00:00:00 2001 From: Matt Jonker Date: Tue, 5 Jan 2021 06:00:12 -0500 Subject: [PATCH 2/4] More closely mimics a real browser, like Python's cloudscraper Removes the ignoring of the Connection header --- hooman.js | 30 +++++++++++++---- lib/outgoing_message.js | 26 ++++++++++++++ lib/user_agent_settings.js | 69 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 lib/outgoing_message.js create mode 100644 lib/user_agent_settings.js diff --git a/hooman.js b/hooman.js index 7a1928e..a771fa2 100644 --- a/hooman.js +++ b/hooman.js @@ -1,15 +1,20 @@ 'use strict'; +require('./lib/outgoing_message'); const got = require('got'); const solveChallenge = require('./lib/core'); const solveCaptcha = require('./lib/captcha'); const delay = require('./lib/delay'); const log = require('./lib/logging'); const { CookieJar } = require('tough-cookie'); +const SSL_OP_ALL = require('constants').SSL_OP_ALL; const UserAgent = require('user-agents'); +const userAgentSettings = require('./lib/user_agent_settings'); const cookieJar = new CookieJar(); // Automatically parse and store cookies const challengeInProgress = {}; +const SIGNATURE_ALGORITHMS = 'ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:ecdsa_secp521r1_sha512:ed25519:ed448:rsa_pss_pss_sha256:rsa_pss_pss_sha384:rsa_pss_pss_sha512:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha256:rsa_pkcs1_sha384:rsa_pkcs1_sha512:ECDSA+SHA224:RSA+SHA224:DSA+SHA224:DSA+SHA256:DSA+SHA384:DSA+SHA512'; + // Got instance to handle cloudflare bypass const instance = got.extend({ cookieJar, @@ -24,13 +29,12 @@ const instance = got.extend({ captchaKey: process.env.HOOMAN_CAPTCHA_KEY, rucaptcha: process.env.HOOMAN_RUCAPTCHA, http2: false, // http2 doesn't work well with proxies - headers: { - 'accept-encoding': 'gzip, deflate', - 'accept-language': 'en-US,en;q=0.5', - 'cache-control': 'max-age=0', - 'upgrade-insecure-requests': '1', - 'user-agent': new UserAgent().toString(), // Use random user agent between sessions - }, // Mimic browser environment + honorCipherOrder: true, + maxVersion: 'TLSv1.3', + sigalgs: SIGNATURE_ALGORITHMS, + ALPNProtocols: ['http/1.1'], + ecdhCurve: 'prime256v1', + secureOptions: SSL_OP_ALL, hooks: { beforeRequest: [ async (options) => { @@ -39,6 +43,18 @@ const instance = got.extend({ await delay(1000); } + if(options.headers['user-agent'] === got.defaults.options.headers['user-agent']) { + const userAgent = new UserAgent(/Firefox|Chrome/); + options.headers['user-agent'] = userAgent.toString(); + } + + const { headers, cipherSuite } = userAgentSettings(options.headers['user-agent']); + options.headers = got.mergeOptions({headers: options.headers}, {headers}).headers; + + if(!options.ciphers) { + options.ciphers = cipherSuite.join(':'); + } + // Add required headers to mimic browser environment options.headers.host = options.url.host; options.headers.origin = options.url.origin; diff --git a/lib/outgoing_message.js b/lib/outgoing_message.js new file mode 100644 index 0000000..1eef998 --- /dev/null +++ b/lib/outgoing_message.js @@ -0,0 +1,26 @@ +var http = require('http'); + +const originalStoreHeader = http.OutgoingMessage.prototype._storeHeader; +http.OutgoingMessage.prototype._storeHeader = function _storeHeader(firstLine, headers) { + originalStoreHeader.bind(this)(firstLine, headers); + const lines = this._header.split('\r\n'); + const command = lines.shift(); + const newHeader = this._header = lines.reduce((result, line) => { + const [ key, value ] = line.split(/:(.+)/); + if (key && value) { + const words = key.split('-'); + const newKey = words.map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('-'); + const newLine = [newKey, value.trim()].join(': '); + if(newKey === 'Host') { + result.unshift(newLine); + } else { + result.push(newLine); + } + } else { + result.push(line); + } + return result; + }, []); + newHeader.unshift(command); + this._header = newHeader.join('\r\n'); +}; \ No newline at end of file diff --git a/lib/user_agent_settings.js b/lib/user_agent_settings.js new file mode 100644 index 0000000..38c7701 --- /dev/null +++ b/lib/user_agent_settings.js @@ -0,0 +1,69 @@ +const HEADERS = { + chrome: { + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', + 'accept-language': 'en-US,en;q=0.9', + 'accept-encoding': 'gzip, deflate, br' + }, + firefox: { + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'accept-language': 'en-US,en;q=0.5', + 'accept-encoding': 'gzip, deflate, br' + } +}; + +const CIPHER_SUITES = { + 'chrome': [ + 'TLS_AES_128_GCM_SHA256', + 'TLS_AES_256_GCM_SHA384', + 'TLS_CHACHA20_POLY1305_SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-CHACHA20-POLY1305', + 'ECDHE-RSA-CHACHA20-POLY1305', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-RSA-AES256-SHA', + 'AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'AES128-SHA', + 'AES256-SHA', + 'DES-CBC3-SHA' + ], + 'firefox': [ + 'TLS_AES_256_GCM_SHA384', + 'TLS_CHACHA20_POLY1305_SHA256', + 'TLS_AES_128_GCM_SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-CHACHA20-POLY1305', + 'ECDHE-RSA-CHACHA20-POLY1305', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES256-SHA', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-RSA-AES256-SHA', + 'DHE-RSA-AES128-SHA', + 'DHE-RSA-AES256-SHA', + 'AES128-SHA', + 'AES256-SHA', + 'DES-CBC3-SHA' + ] +}; + +module.exports = (userAgent) => { + const match = userAgent.match(/(firefox|chrome)[/\s]([\d.]+)/i); + if(!match) { + throw 'Unsupported user agent: ' + userAgent; + } + + const browser = match[1].toLowerCase(); + const cipherSuite = CIPHER_SUITES[browser]; + const headers = HEADERS[browser]; + headers['user-agent'] = userAgent; + return { + cipherSuite, + headers + }; +}; From 5224066c3c85719a3d53201ad624d72957e2d7f9 Mon Sep 17 00:00:00 2001 From: Matt Jonker Date: Sat, 9 Jan 2021 05:40:32 -0500 Subject: [PATCH 3/4] Removes OutgoingMessage monkey-patching Splits out distinct concerns into instances and uses composes the final instance from them Removes SSL emulation if Node < 12 is being used --- hooman.js | 121 +++++++++++++++++++++++++++++-------- lib/outgoing_message.js | 26 -------- lib/user_agent_settings.js | 4 +- 3 files changed, 98 insertions(+), 53 deletions(-) delete mode 100644 lib/outgoing_message.js diff --git a/hooman.js b/hooman.js index a771fa2..5eec3bc 100644 --- a/hooman.js +++ b/hooman.js @@ -1,5 +1,4 @@ 'use strict'; -require('./lib/outgoing_message'); const got = require('got'); const solveChallenge = require('./lib/core'); const solveCaptcha = require('./lib/captcha'); @@ -15,8 +14,90 @@ const challengeInProgress = {}; const SIGNATURE_ALGORITHMS = 'ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:ecdsa_secp521r1_sha512:ed25519:ed448:rsa_pss_pss_sha256:rsa_pss_pss_sha384:rsa_pss_pss_sha512:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha256:rsa_pkcs1_sha384:rsa_pkcs1_sha512:ECDSA+SHA224:RSA+SHA224:DSA+SHA224:DSA+SHA256:DSA+SHA384:DSA+SHA512'; +const userAgentSsl = got.extend( + { + honorCipherOrder: true, + maxVersion: 'TLSv1.3', + sigalgs: SIGNATURE_ALGORITHMS, + ALPNProtocols: ['http/1.1'], + ecdhCurve: 'prime256v1', + secureOptions: SSL_OP_ALL, + hooks: { + beforeRequest: [ + async (options) => { + try { + const { cipherSuite } = userAgentSettings(options.headers['user-agent']); + + if (!options.ciphers) { + options.ciphers = cipherSuite.join(':'); + } + } + catch(e) { + console.warn('Unable to set user agent ciphers: ' + e); + } + }, + ] + } + } +); + +const ensureUserAgent = got.extend({ + hooks: { + beforeRequest: [ + async (options) => { + if (options.headers['user-agent'] === got.defaults.options.headers['user-agent']) { + const userAgent = new UserAgent(/Firefox|Chrome/); + options.headers['user-agent'] = userAgent.toString(); + } + } + ] + } +}); + +const userAgentHeaders = got.extend({ + hooks: { + beforeRequest: [ + async (options) => { + try { + const { headers } = userAgentSettings(options.headers['user-agent']); + options.headers = got.mergeOptions({ headers: options.headers }, { headers }).headers; + } + catch(e) { + console.warn('Unable to set user agent headers: ' + e); + } + + // Add required headers to mimic browser environment + options.headers.host = options.url.host; + options.headers.origin = options.url.origin; + options.headers.referer = options.url.href; + }, + ] + } +}); + +const capitalizedHeaders = got.extend({ + hooks: { + beforeRequest: [ + async (options) => { + options.headers = Object.keys(options.headers).reduce((result, lowerCaseKey) => { + const value = options.headers[lowerCaseKey]; + const words = lowerCaseKey.split('-'); + const upperCaseKey = words.map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('-'); + const header = [upperCaseKey, value]; + if(upperCaseKey === 'Host') { + result.unshift(header); + } else { + result.push(header); + } + return result; + }, []); + }, + ] + } +}); + // Got instance to handle cloudflare bypass -const instance = got.extend({ +const captcha = got.extend({ cookieJar, retry: { limit: 2, @@ -29,12 +110,6 @@ const instance = got.extend({ captchaKey: process.env.HOOMAN_CAPTCHA_KEY, rucaptcha: process.env.HOOMAN_RUCAPTCHA, http2: false, // http2 doesn't work well with proxies - honorCipherOrder: true, - maxVersion: 'TLSv1.3', - sigalgs: SIGNATURE_ALGORITHMS, - ALPNProtocols: ['http/1.1'], - ecdhCurve: 'prime256v1', - secureOptions: SSL_OP_ALL, hooks: { beforeRequest: [ async (options) => { @@ -42,23 +117,6 @@ const instance = got.extend({ while (challengeInProgress[options.url.host]) { await delay(1000); } - - if(options.headers['user-agent'] === got.defaults.options.headers['user-agent']) { - const userAgent = new UserAgent(/Firefox|Chrome/); - options.headers['user-agent'] = userAgent.toString(); - } - - const { headers, cipherSuite } = userAgentSettings(options.headers['user-agent']); - options.headers = got.mergeOptions({headers: options.headers}, {headers}).headers; - - if(!options.ciphers) { - options.ciphers = cipherSuite.join(':'); - } - - // Add required headers to mimic browser environment - options.headers.host = options.url.host; - options.headers.origin = options.url.origin; - options.headers.referer = options.url.href; }, ], afterResponse: [ @@ -153,4 +211,17 @@ const instance = got.extend({ mutableDefaults: true, // Defines if config can be changed later }); +const nodeMajorVersion = Number(process.versions.node.split('.')[0]); +const instances = [captcha, ensureUserAgent, userAgentHeaders]; +if(nodeMajorVersion >= 12) { + instances.push(userAgentSsl); +} +else { + console.warn('User agent SSL emulation is only available in Node v12+'); +} +// header capitalization should come last, because it changes headers from an object to an array +instances.push(capitalizedHeaders); + +const instance = got.extend(...instances); + module.exports = instance; diff --git a/lib/outgoing_message.js b/lib/outgoing_message.js deleted file mode 100644 index 1eef998..0000000 --- a/lib/outgoing_message.js +++ /dev/null @@ -1,26 +0,0 @@ -var http = require('http'); - -const originalStoreHeader = http.OutgoingMessage.prototype._storeHeader; -http.OutgoingMessage.prototype._storeHeader = function _storeHeader(firstLine, headers) { - originalStoreHeader.bind(this)(firstLine, headers); - const lines = this._header.split('\r\n'); - const command = lines.shift(); - const newHeader = this._header = lines.reduce((result, line) => { - const [ key, value ] = line.split(/:(.+)/); - if (key && value) { - const words = key.split('-'); - const newKey = words.map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('-'); - const newLine = [newKey, value.trim()].join(': '); - if(newKey === 'Host') { - result.unshift(newLine); - } else { - result.push(newLine); - } - } else { - result.push(line); - } - return result; - }, []); - newHeader.unshift(command); - this._header = newHeader.join('\r\n'); -}; \ No newline at end of file diff --git a/lib/user_agent_settings.js b/lib/user_agent_settings.js index 38c7701..ab3fa7f 100644 --- a/lib/user_agent_settings.js +++ b/lib/user_agent_settings.js @@ -12,7 +12,7 @@ const HEADERS = { }; const CIPHER_SUITES = { - 'chrome': [ + chrome: [ 'TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256', @@ -30,7 +30,7 @@ const CIPHER_SUITES = { 'AES256-SHA', 'DES-CBC3-SHA' ], - 'firefox': [ + firefox: [ 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_AES_128_GCM_SHA256', From c22b00d7aeafbdfbc3f637193fcdb381fb9cb01c Mon Sep 17 00:00:00 2001 From: Matt Jonker Date: Sat, 9 Jan 2021 05:44:52 -0500 Subject: [PATCH 4/4] Uses log --- hooman.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooman.js b/hooman.js index 5eec3bc..20810e2 100644 --- a/hooman.js +++ b/hooman.js @@ -33,7 +33,7 @@ const userAgentSsl = got.extend( } } catch(e) { - console.warn('Unable to set user agent ciphers: ' + e); + log.warn('Unable to set user agent ciphers: ' + e); } }, ] @@ -63,7 +63,7 @@ const userAgentHeaders = got.extend({ options.headers = got.mergeOptions({ headers: options.headers }, { headers }).headers; } catch(e) { - console.warn('Unable to set user agent headers: ' + e); + log.warn('Unable to set user agent headers: ' + e); } // Add required headers to mimic browser environment @@ -217,7 +217,7 @@ if(nodeMajorVersion >= 12) { instances.push(userAgentSsl); } else { - console.warn('User agent SSL emulation is only available in Node v12+'); + log.warn('User agent SSL emulation is only available in Node v12+'); } // header capitalization should come last, because it changes headers from an object to an array instances.push(capitalizedHeaders);