From 634e06b779c2fa9f6964510d809cf534be7e324b Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Fri, 20 Jul 2012 08:24:24 +0200 Subject: [PATCH] euscan: don't fail if robots.txt can't be read Signed-off-by: Corentin Chary --- pym/euscan/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index 17df9f4..0af9315 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -416,7 +416,7 @@ def urlallowed(url): setdefaulttimeout(timeout) - return rp.can_fetch(CONFIG['user-agent'], url) if rp else False + return rp.can_fetch(CONFIG['user-agent'], url) if rp else True def urlopen(url, timeout=None, verb="GET"):