diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index f5ee6ee..987e105 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -20,7 +20,8 @@ CONFIG = { 'scan-dir': True, 'oneshot': False, 'user-agent' : 'escan (http://euscan.iksaif.net)', - 'skip-robots-txt' : False + 'skip-robots-txt' : False, + 'cache' : False } output = EOutput(CONFIG['quiet']) diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 6412220..1dea1b3 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -267,11 +267,15 @@ def urlopen(url, timeout=None, verb="GET"): request.add_header('User-Agent', CONFIG['user-agent']) + handlers = [] + + if CONFIG['cache']: + from cache import CacheHandler + handlers.append(CacheHandler(CONFIG['cache'])) + if CONFIG['verbose']: debuglevel = CONFIG['verbose'] - 1 - handlers = [urllib2.HTTPHandler(debuglevel=debuglevel)] - else: - handlers = [] + handlers.append(urllib2.HTTPHandler(debuglevel=debuglevel)) opener = urllib2.build_opener(*handlers) diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 0903fc1..ecda09b 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -20,6 +20,7 @@ def filter_versions(cp, versions): for url, version in versions: + ''' Try to keep the most specific urls (determinted by the length) ''' if version in filtered and len(url) < len(filtered[version]): continue