Changed WebpageResolver default arguements, added more robust sysargs, added transparent to ruleset

Author: cc-d

bin/resolveimg.py

@@ -3,6 +3,7 @@
 import sys
 import imageresolver
 import logging
+import time
 from optparse import OptionParser
 
 logger = logging.getLogger('ImageResolver')
@@ -14,6 +15,11 @@
 opts.add_option("-r","--max-read", dest="max_read",help="Set the max read size")
 opts.add_option("-c","--chunk-size",dest="chunk_size",help="Chunk size to read on each pass")
 opts.add_option("-a","--read-all",dest="read_all",help="Read the entire image before checking size. Useful for some JPGs. Overrides --max-read")
+opts.add_option("--no-adblock", action="store_true",dest="use_adblock_filters",help="Do not use whitelist.txt or blacklist.txt adblock filters")
+opts.add_option("--no-ruleset", action="store_true",dest="use_js_ruleset",help="Do not use a custom ruleset for scoring.")
+opts.add_option("--benchmark", action="store_true",dest="benchmark",help="Benchmark the total time it takes for the script to return an image")
+opts.add_option("-l","--load-images", action="store_true",dest="load_images",help="Load images")
+opts.add_option("-p","--parser", dest="parser",help="Choose a parser to use")
 
 (options,args) = opts.parse_args()
 
@@ -26,7 +32,17 @@
 if options.chunk_size:
 	kw_options['chunk_size'] = int(options.chunk_size)
 
+if options.use_adblock_filters:
+	kw_options['use_adblock_filters'] = False
+
+if options.use_js_ruleset:
+	kw_options['use_js_ruleset'] = False
+
+if options.parser:
+	kw_options['parser'] = options.parser
+
 kw_options['debug'] = options.debug
+kw_options['load_images'] = options.load_images
 
 try:
 	url = args[0]
@@ -39,10 +55,17 @@
 	print "URL required. Please use the url option or pass a url as the first argument"
 	sys.exit(-1)
 
+	
+if options.benchmark:
+	t1 = time.time()
+
 i = imageresolver.ImageResolver(**kw_options)
 i.register(imageresolver.FileExtensionResolver())
 i.register(imageresolver.PluginResolver())
-i.register(imageresolver.WebpageResolver(load_images=True, parser='lxml'))
+i.register(imageresolver.WebpageResolver(**kw_options))
 
 print i.resolve(url)
 
+if options.benchmark:
+	print 'TOTAL TIME', time.time() - t1
+

imageresolver/__init__.py

@@ -169,8 +169,8 @@ def resolve(self,url,**kwargs):
 
 class WebpageResolver(object):
 	def __init__(self,**kwargs):
-		self.load_images = kwargs.get('load_images',False)
-		self.use_js_ruleset = kwargs.get('use_js_ruleset',False)
+		self.load_images = kwargs.get('load_images',True)
+		self.use_js_ruleset = kwargs.get('use_js_ruleset',True)
 		self.use_adblock_filters = kwargs.get('use_adblock_filters',True)
 		self.significant_surface = kwargs.get('significant_surface', 100*100)
 
@@ -224,6 +224,7 @@ def _score(self,image):
 				{'pattern':'1x1','score':-1},
 				{'pattern':'pixel','score':-1},
 				{'pattern':'ads','score':-1},
+				{'pattern':'transparent','score':-1}
 			]
 
 			for r in rules: