From d7f655cdde691c4afd1b5db55134bbe24970fa40 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Sun, 2 Oct 2011 10:04:44 +0200 Subject: [PATCH] euscan: add an optional persistent cache Signed-off-by: Corentin Chary --- pym/euscan/__init__.py | 3 ++- pym/euscan/helpers.py | 10 +++++++--- pym/euscan/scan.py | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index f5ee6ee..987e105 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -20,7 +20,8 @@ CONFIG = { 'scan-dir': True, 'oneshot': False, 'user-agent' : 'escan (http://euscan.iksaif.net)', - 'skip-robots-txt' : False + 'skip-robots-txt' : False, + 'cache' : False } output = EOutput(CONFIG['quiet']) diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 6412220..1dea1b3 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -267,11 +267,15 @@ def urlopen(url, timeout=None, verb="GET"): request.add_header('User-Agent', CONFIG['user-agent']) + handlers = [] + + if CONFIG['cache']: + from cache import CacheHandler + handlers.append(CacheHandler(CONFIG['cache'])) + if CONFIG['verbose']: debuglevel = CONFIG['verbose'] - 1 - handlers = [urllib2.HTTPHandler(debuglevel=debuglevel)] - else: - handlers = [] + handlers.append(urllib2.HTTPHandler(debuglevel=debuglevel)) opener = urllib2.build_opener(*handlers) diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 0903fc1..ecda09b 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -20,6 +20,7 @@ def filter_versions(cp, versions): for url, version in versions: + ''' Try to keep the most specific urls (determinted by the length) ''' if version in filtered and len(url) < len(filtered[version]): continue