fixed issues with OpenGraph didnt return a real image. Fix tests

Author: cfreak-dev

imageresolver/__main__.py

@@ -44,9 +44,6 @@
 if options.load_images:
     kw_options['load_images'] = False
 
-if options.get_info:
-    kw_options['get_info'] = True
-
 kw_options['use_adblock_filters'] = options.use_adblock_filters
 kw_options['debug'] = options.debug
 

imageresolver/base.py

@@ -120,7 +120,7 @@ def next(self, filters, url, callback):
     def resolve(self, url):
         logger.debug('Attempting to resolve ' + str(url))
         for f in self.filters:
-            resp = f.resolve(url, debug=self.debug)
+            resp = f.resolve(url)
 
             # returns the first filter that gives us something
             if resp:

imageresolver/plugins/imgur.py

@@ -3,33 +3,33 @@
 import logging
 
 from future.standard_library import install_aliases
+
 install_aliases()
 from urllib.parse import urlparse
 
 
 class Plugin(object):
-	def get_image(self, url, soup):
-		if re.search('http(s*):\/\/(i\.|m\.)*imgur.com\/(gallery\/){0,1}(.*)', url):
-			logger = logging.getLogger('ImageResolver')
-			logger.debug('Resolving using plugin {} {}'.format(os.path.basename(__file__), url))
-			parsed = urlparse(url)
+    def get_image(self, url, soup):
+        if re.search('http(s*):\/\/(i\.|m\.)*imgur.com\/(gallery\/){0,1}(.*)', url):
+            logger = logging.getLogger('ImageResolver')
+            logger.debug('Resolving using plugin {} {}'.format(os.path.basename(__file__), url))
+            parsed = urlparse(url)
 
-			if parsed.path[1:8] == 'gallery':
-				logger.debug('Detected imgur gallery.')
-				tag = soup.find('div', {'id': '1', 'class': 'album-image'})
-				image = re.findall('i\.imgur.com\/.*\.\w+', str(tag))
-				if len(image) >= 1:
-					return 'https://' + image[0]
-			
-			elif parsed.path[0:3] == '/a/':
-				logger.debug('Detected imgur album.')
-				tag = soup.find('meta', {'name': 'twitter:image0:src'})
-				if tag:
-					return tag['content']
+            if parsed.path[1:8] == 'gallery':
+                logger.debug('Detected imgur gallery.')
+                tag = soup.find('div', {'id': '1', 'class': 'album-image'})
+                image = re.findall('i\.imgur.com\/.*\.\w+', str(tag))
+                if len(image) >= 1:
+                    return 'https://' + image[0]
 
-			else:
-				parsed = urlparse(url)
-				if re.search('imgur.com(:80)*', parsed.netloc) and os.path.basename(parsed.path):
-					return 'https://i.imgur.com/' + os.path.basename(parsed.path) + '.jpg'
-		return None
+            elif parsed.path[0:3] == '/a/':
+                logger.debug('Detected imgur album.')
+                tag = soup.find('meta', {'name': 'twitter:image0:src'})
+                if tag:
+                    return tag['content']
 
+            else:
+                parsed = urlparse(url)
+                if re.search('imgur.com(:80)*', parsed.netloc) and os.path.basename(parsed.path):
+                    return 'https://i.imgur.com/' + os.path.basename(parsed.path) + '.jpg'
+        return None

imageresolver/plugins/opengraph.py

@@ -2,7 +2,7 @@
 import os
 import logging
 from operator import itemgetter
-
+from imageresolver import FileExtensionResolver
 
 class Plugin(object):
     def get_image(self, url, soup):
@@ -20,8 +20,10 @@ def get_image(self, url, soup):
             tags = soup.find_all('meta', {ogtag['attribute']: ogtag['name']})
             if tags:
                 try:
-                    ogimages.extend([{'url': image[ogtag['value']], 'type': ogtag['type'], 'score': 0}
-                                     for image in tags])
+                    for image in tags:
+                        url = FileExtensionResolver().resolve(image['url'])
+                        if url:
+                            ogimages.append({'url': url, 'type': ogtag['type'], 'score': 0})
                 except KeyError as e:
                     pass
 
@@ -35,10 +37,15 @@ def get_image(self, url, soup):
                 resolved_image = ogimages[0]['url']
             else:
                 for image in ogimages:
-                    if re.search('(large|big)', image['url'], re.IGNORECASE):
-                        image['score'] += 1
-                    if image['type'] == 'twitter':
-                        image['score'] += 1
+                    # sometimes opengraph tags don't have an actual image?
+                    url = FileExtensionResolver().resolve(image['url'])
+                    if not url:
+                        image['score'] = -1
+                    else:
+                        if re.search('(large|big)', image['url'], re.IGNORECASE):
+                            image['score'] += 1
+                        if image['type'] == 'twitter':
+                            image['score'] += 1
 
                 ogimages.sort(key=itemgetter('score'), reverse=True)
                 resolved_image = ogimages[0]['url']

tests/test_imageresolver.py

@@ -1,49 +1,41 @@
-import sys
+from __future__ import absolute_import
 import unittest
-import requests
-from os.path import dirname,abspath
-sys.path.append( dirname( dirname( dirname( abspath(__file__)) ) ) )
-from imageresolver import ImageResolver, FileExtensionResolver, PluginResolver, WebpageResolver
+from imageresolver import ImageResolver, FileExtensionResolver, WebpageResolver
+
 
 class TestImageResolver(unittest.TestCase):
 	def setUp(self):
 		# set to an imgur page
-		self.imgur_page = 'http://imgur.com/adtBv9Y'
+		self.imgur_page = 'https://imgur.com/adtBv9Y'
 
 		# set to the expected result of the imgur page
 		# also checks the file extension
-		self.imgur_result = 'http://i.imgur.com/adtBv9Y.jpg'
+		self.imgur_result = 'https://i.imgur.com/adtBv9Y.jpg'
 
 		# set to a web url
-		self.web_url = 'http://xkcd.com/353/'
+		self.web_url = 'https://xkcd.com/353/'
 
 		# set to the expected return image from the web url
-		self.web_img = 'http://imgs.xkcd.com/comics/python.png'
+		self.web_img = 'https://imgs.xkcd.com/comics/python.png'
 
 	def test_fetch_image_info(self):
 		i = ImageResolver()
-		(ext,width,height) = i.fetch_image_info(self.web_img)
+		(ext, width, height) = i.fetch_image_info(self.web_img)
 
-		self.assertEquals(ext,'.png')
-		self.assertEquals(width,518)
-		self.assertEquals(height,588)
+		self.assertEquals(ext, '.png')
+		self.assertEquals(width, 518)
+		self.assertEquals(height, 588)
 
-	def test_resolve_plugin(self):
-		i = ImageResolver()
-		i.register(PluginResolver())
-		src = i.resolve(self.imgur_page)
-		self.assertEquals(src,self.imgur_result)
-	
 	def test_resolve_fileext(self):
 		i = ImageResolver()
 		i.register(FileExtensionResolver())
 		src = i.resolve(self.web_img)
 
-		self.assertEquals(src,self.web_img)
+		self.assertEquals(src, self.web_img)
 
 	def test_resolve_webpage(self):
 		i = ImageResolver()
 		i.register(WebpageResolver(load_images=True))
 		src = i.resolve(self.web_url)
-		self.assertEquals(src, self.web_img )
+		self.assertEquals(src, self.web_img)