euscan: add an optional persistent cache

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

pym/euscan/__init__.py

@@ -20,7 +20,8 @@ CONFIG = {
     'scan-dir': True,
     'oneshot': False,
     'user-agent' : 'escan (http://euscan.iksaif.net)',
-    'skip-robots-txt' : False
+    'skip-robots-txt' : False,
+    'cache' : False
 }
 
 output = EOutput(CONFIG['quiet'])

pym/euscan/helpers.py

@@ -267,11 +267,15 @@ def urlopen(url, timeout=None, verb="GET"):
 
     request.add_header('User-Agent', CONFIG['user-agent'])
 
+    handlers = []
+
+    if CONFIG['cache']:
+        from cache import CacheHandler
+        handlers.append(CacheHandler(CONFIG['cache']))
+
     if CONFIG['verbose']:
         debuglevel = CONFIG['verbose'] - 1
-        handlers = [urllib2.HTTPHandler(debuglevel=debuglevel)]
-    else:
-        handlers = []
+        handlers.append(urllib2.HTTPHandler(debuglevel=debuglevel))
 
     opener = urllib2.build_opener(*handlers)
 

pym/euscan/scan.py

@@ -20,6 +20,7 @@ def filter_versions(cp, versions):
 
     for url, version in versions:
 
+
         ''' Try to keep the most specific urls (determinted by the length) '''
         if version in filtered and len(url) < len(filtered[version]):
             continue