euscan: show a trace when blocked by robots.txt

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
2011-10-08 08:33:48 +02:00 · 2011-10-08 08:33:48 +02:00 · 8be69f2350
commit 8be69f2350
parent 739358cf7c
1 changed files with 1 additions and 0 deletions
--- a/pym/euscan/helpers.py
+++ b/pym/euscan/helpers.py
@ -253,6 +253,7 @@ def urlallowed(url):

 def urlopen(url, timeout=None, verb="GET"):
    if not urlallowed(url):
+        output.einfo("Url '%s' blocked by robots.txt" % url)
        return None

    if not timeout: