Added scoring to opengraph plugin, changed default options.

Author: cc-d

bin/resolveimg.py

@@ -15,15 +15,16 @@
 opts.add_option("-r","--max-read", dest="max_read",help="Set the max read size")
 opts.add_option("-c","--chunk-size",dest="chunk_size",help="Chunk size to read on each pass")
 opts.add_option("-a","--read-all",dest="read_all",help="Read the entire image before checking size. Useful for some JPGs. Overrides --max-read")
-opts.add_option("--no-adblock", action="store_true",dest="use_adblock_filters",help="Do not use whitelist.txt or blacklist.txt adblock filters")
-opts.add_option("--no-ruleset", action="store_true",dest="use_js_ruleset",help="Do not use a custom ruleset for scoring.")
+opts.add_option("-b","--adblock", action="store_true",dest="use_adblock_filters",help="Use adblock filters.")
+opts.add_option("-s","--no-ruleset", action="store_true",dest="use_js_ruleset",help="Use a custom ruleset for scoring.")
 opts.add_option("--benchmark", action="store_true",dest="benchmark",help="Benchmark the total time it takes for the script to return an image")
-opts.add_option("-l","--load-images", action="store_true",dest="load_images",help="Load images")
+opts.add_option("-n","--no-load-images", action="store_true",dest="load_images",help="Do not load images")
 opts.add_option("-p","--parser", dest="parser",help="Choose a parser to use")
 
 (options,args) = opts.parse_args()
 
 kw_options = {}
+
 if options.read_all:
 	kw_options['read_all'] = True
 elif options.max_read:
@@ -32,17 +33,17 @@
 if options.chunk_size:
 	kw_options['chunk_size'] = int(options.chunk_size)
 
-if options.use_adblock_filters:
-	kw_options['use_adblock_filters'] = False
-
 if options.use_js_ruleset:
 	kw_options['use_js_ruleset'] = False
 
 if options.parser:
 	kw_options['parser'] = options.parser
 
+if options.load_images:
+	kw_options['load_images'] = True
+
+kw_options['use_adblock_filters'] = options.use_adblock_filters
 kw_options['debug'] = options.debug
-kw_options['load_images'] = options.load_images
 
 try:
 	url = args[0]

imageresolver/__init__.py

@@ -153,7 +153,7 @@ class WebpageResolver(object):
 	def __init__(self,**kwargs):
 		self.load_images = kwargs.get('load_images',True)
 		self.use_js_ruleset = kwargs.get('use_js_ruleset',True)
-		self.use_adblock_filters = kwargs.get('use_adblock_filters',True)
+		self.use_adblock_filters = kwargs.get('use_adblock_filters',False)
 		self.significant_surface = kwargs.get('significant_surface', 100*100)
 
 		cwd = os.path.dirname(__file__)

imageresolver/plugins/opengraph.py

@@ -2,29 +2,48 @@
 import os
 import logging
 from bs4 import BeautifulSoup
+from operator import itemgetter
 
 class Plugin:
 	def get_image(self, url, soup):
 
-		ogtags = [{'attribute':'property', 'name':'og:image', 'value':'content'},
-			{'attribute':'rel', 'name':'image_src', 'value':'href'},
-			{'attribute':'name', 'name':'twitter:image', 'value':'value'},
-			{'attribute':'name', 'name':'twitter:image', 'value':'content'}]
+		ogtags = [{'type':'facebook','attribute':'property', 'name':'og:image', 'value':'content'},
+			{'type':'facebook','attribute':'rel', 'name':'image_src', 'value':'href'},
+			{'type':'twitter','attribute':'name', 'name':'twitter:image', 'value':'value'},
+			{'type':'twitter','attribute':'name', 'name':'twitter:image', 'value':'content'},
+			{'type':'twitter','attribute':'property', 'name':'twitter:image', 'value':'content'},
+			{'type':'image','attribute':'itemprop', 'name':'image', 'value':'content'}]
 
 		ogimages = []		
 
 		for ogtag in ogtags:
 			tags = soup.find_all('meta', {ogtag['attribute']:ogtag['name']})
 			if tags != []:
-				try:
-					ogimages = ogimages + [image[ogtag['value']] for image in tags]
-				except KeyError:
-					pass
+				for image in tags:
+					try:
+						ogimages = ogimages + [{'url':image[ogtag['value']], 'type':ogtag['type'], 'score':0} for image in tags]
+					except KeyError as e:
+						pass
 
-		if len(ogimages) >= 1:
-			logger = logging.getLogger('ImageResolver')
-			logger.debug('Resolving using plugin ' + str(os.path.basename(__file__)) + ' ' +  str(url))
-			return ogimages[0]
+		ogimages_len = len(ogimages)
+
+		# if more than 1 image, score and return the best one
+		if ogimages_len >= 1:
+			if ogimages_len == 1:
+				logger = logging.getLogger('ImageResolver')
+				logger.debug('Resolving using plugin ' + str(os.path.basename(__file__)) + ' ' +  str(url))
+				return ogimages[0]['url']
+			else:
+				for image in ogimages:
+					if re.search('(large|big)', image['url'], re.IGNORECASE):
+						image['score'] += 1
+					if image['type'] == 'twitter':
+						image['score'] += 1
+
+				ogimages.sort(key=itemgetter('score'), reverse=True)
+				
+				return ogimages[0]['url']
+				
 
 		return None