Improved User Agent emulation

hooman.js

@@ -5,13 +5,99 @@ const solveCaptcha = require('./lib/captcha');
 const delay = require('./lib/delay');
 const log = require('./lib/logging');
 const { CookieJar } = require('tough-cookie');
+const SSL_OP_ALL = require('constants').SSL_OP_ALL;
 const UserAgent = require('user-agents');
+const userAgentSettings = require('./lib/user_agent_settings');
 
 const cookieJar = new CookieJar(); // Automatically parse and store cookies
 const challengeInProgress = {};
 
+const SIGNATURE_ALGORITHMS = 'ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:ecdsa_secp521r1_sha512:ed25519:ed448:rsa_pss_pss_sha256:rsa_pss_pss_sha384:rsa_pss_pss_sha512:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha256:rsa_pkcs1_sha384:rsa_pkcs1_sha512:ECDSA+SHA224:RSA+SHA224:DSA+SHA224:DSA+SHA256:DSA+SHA384:DSA+SHA512';
+
+const userAgentSsl = got.extend(
+  {
+    honorCipherOrder: true,
+    maxVersion: 'TLSv1.3',
+    sigalgs: SIGNATURE_ALGORITHMS,
+    ALPNProtocols: ['http/1.1'],
+    ecdhCurve: 'prime256v1',
+    secureOptions: SSL_OP_ALL,
+    hooks: {
+      beforeRequest: [
+        async (options) => {
+          try {
+            const { cipherSuite } = userAgentSettings(options.headers['user-agent']);
+
+            if (!options.ciphers) {
+              options.ciphers = cipherSuite.join(':');
+            }
+          }
+          catch(e) {
+            log.warn('Unable to set user agent ciphers: ' + e);
+          }
+        },
+      ]
+    }
+  }
+);
+
+const ensureUserAgent = got.extend({
+  hooks: {
+    beforeRequest: [
+      async (options) => {
+        if (options.headers['user-agent'] === got.defaults.options.headers['user-agent']) {
+          const userAgent = new UserAgent(/Firefox|Chrome/);
+          options.headers['user-agent'] = userAgent.toString();
+        }
+      }
+    ]
+  }
+});
+
+const userAgentHeaders = got.extend({
+  hooks: {
+    beforeRequest: [
+      async (options) => {
+        try {
+          const { headers } = userAgentSettings(options.headers['user-agent']);
+          options.headers = got.mergeOptions({ headers: options.headers }, { headers }).headers;
+        }
+        catch(e) {
+          log.warn('Unable to set user agent headers: ' + e);
+        }
+
+        // Add required headers to mimic browser environment
+        options.headers.host = options.url.host;
+        options.headers.origin = options.url.origin;
+        options.headers.referer = options.url.href;
+      },
+    ]
+  }
+});
+
+const capitalizedHeaders = got.extend({
+  hooks: {
+    beforeRequest: [
+      async (options) => {
+        options.headers = Object.keys(options.headers).reduce((result, lowerCaseKey) => {
+          const value = options.headers[lowerCaseKey];
+          const words = lowerCaseKey.split('-');
+          const upperCaseKey = words.map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('-');
+          const header = [upperCaseKey, value];
+          if(upperCaseKey === 'Host') {
+            result.unshift(header);
+          } else {
+            result.push(header);
+          }
+          return result;
+        }, []);
+      },
+    ]
+  }
+});
+
 // Got instance to handle cloudflare bypass
-const instance = got.extend({
+const captcha = got.extend({
   cookieJar,
   retry: {
     limit: 2,
@@ -24,25 +110,13 @@ const instance = got.extend({
   captchaKey: process.env.HOOMAN_CAPTCHA_KEY,
   rucaptcha: process.env.HOOMAN_RUCAPTCHA,
   http2: false, // http2 doesn't work well with proxies
-  headers: {
-    'accept-encoding': 'gzip, deflate',
-    'accept-language': 'en-US,en;q=0.5',
-    'cache-control': 'max-age=0',
-    'upgrade-insecure-requests': '1',
-    'user-agent': new UserAgent().toString(), // Use random user agent between sessions
-  }, // Mimic browser environment
   hooks: {
     beforeRequest: [
       async (options) => {
         // In case the host is in challenge progress wait
         while (challengeInProgress[options.url.host]) {
           await delay(1000);
         }
-
-        // Add required headers to mimic browser environment
-        options.headers.host = options.url.host;
-        options.headers.origin = options.url.origin;
-        options.headers.referer = options.url.href;
       },
     ],
     afterResponse: [
@@ -137,4 +211,17 @@ const instance = got.extend({
   mutableDefaults: true, // Defines if config can be changed later
 });
 
+const nodeMajorVersion = Number(process.versions.node.split('.')[0]);
+const instances = [captcha, ensureUserAgent, userAgentHeaders];
+if(nodeMajorVersion >= 12) {
+  instances.push(userAgentSsl);
+}
+else {
+  log.warn('User agent SSL emulation is only available in Node v12+');
+}
+// header capitalization should come last, because it changes headers from an object to an array
+instances.push(capitalizedHeaders);
+
+const instance = got.extend(...instances);
+
 module.exports = instance;

lib/captcha.js

@@ -35,11 +35,10 @@ const solve = async (response) => {
           body.push('h-captcha-response=' + captchaResponse);
         }
 
-        const baseUrl = response.url.substring(0, response.url.lastIndexOf('/'));
         const { method, action, enctype } = document.getElementById('challenge-form');
         return {
           method,
-          url: action.startsWith('/') ? baseUrl + action : action,
+          url: new URL(action, response.url),
           headers: {
             'content-type': enctype, // application/x-www-form-urlencoded
           },

lib/user_agent_settings.js

@@ -0,0 +1,69 @@
+const HEADERS = {
+  chrome: { 
+    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+    'accept-language': 'en-US,en;q=0.9', 
+    'accept-encoding': 'gzip, deflate, br'
+  },
+  firefox: {
+    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'accept-language': 'en-US,en;q=0.5',
+    'accept-encoding': 'gzip, deflate, br'
+  }
+};
+
+const CIPHER_SUITES = {
+  chrome: [
+    'TLS_AES_128_GCM_SHA256',
+    'TLS_AES_256_GCM_SHA384',
+    'TLS_CHACHA20_POLY1305_SHA256',
+    'ECDHE-ECDSA-AES128-GCM-SHA256',
+    'ECDHE-RSA-AES128-GCM-SHA256',
+    'ECDHE-ECDSA-AES256-GCM-SHA384',
+    'ECDHE-RSA-AES256-GCM-SHA384',
+    'ECDHE-ECDSA-CHACHA20-POLY1305',
+    'ECDHE-RSA-CHACHA20-POLY1305',
+    'ECDHE-RSA-AES128-SHA',
+    'ECDHE-RSA-AES256-SHA',
+    'AES128-GCM-SHA256',
+    'AES256-GCM-SHA384',
+    'AES128-SHA',
+    'AES256-SHA',
+    'DES-CBC3-SHA'
+  ],
+  firefox: [
+    'TLS_AES_256_GCM_SHA384',
+    'TLS_CHACHA20_POLY1305_SHA256',
+    'TLS_AES_128_GCM_SHA256',
+    'ECDHE-ECDSA-AES128-GCM-SHA256',
+    'ECDHE-RSA-AES128-GCM-SHA256',
+    'ECDHE-ECDSA-CHACHA20-POLY1305',
+    'ECDHE-RSA-CHACHA20-POLY1305',
+    'ECDHE-ECDSA-AES256-GCM-SHA384',
+    'ECDHE-RSA-AES256-GCM-SHA384',
+    'ECDHE-ECDSA-AES256-SHA',
+    'ECDHE-ECDSA-AES128-SHA',
+    'ECDHE-RSA-AES128-SHA',
+    'ECDHE-RSA-AES256-SHA',
+    'DHE-RSA-AES128-SHA',
+    'DHE-RSA-AES256-SHA',
+    'AES128-SHA',
+    'AES256-SHA',
+    'DES-CBC3-SHA'
+  ]
+};
+
+module.exports = (userAgent) => {
+  const match = userAgent.match(/(firefox|chrome)[/\s]([\d.]+)/i);
+  if(!match) {
+    throw 'Unsupported user agent: ' + userAgent;
+  }
+
+  const browser = match[1].toLowerCase();
+  const cipherSuite = CIPHER_SUITES[browser];
+  const headers = HEADERS[browser];
+  headers['user-agent'] = userAgent;
+  return {
+    cipherSuite,
+    headers
+  };
+};