euscan: show a trace when blocked by robots.txt

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary committed 2011-10-08 08:33:48 +02:00
1 parent 739358cf7c
commit 8be69f2350
1 file changed
+1
+1
View File
@@ -253,6 +253,7 @@ def urlallowed(url):
def urlopen(url, timeout=None, verb="GET"):
if not urlallowed(url):
output.einfo("Url '%s' blocked by robots.txt" % url)
return None
if not timeout: