euscan: show a trace when blocked by robots.txt
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
1 parent
739358cf7c
commit
8be69f2350
1 file changed
+1
@@ -253,6 +253,7 @@ def urlallowed(url):
|
||||
|
||||
def urlopen(url, timeout=None, verb="GET"):
|
||||
if not urlallowed(url):
|
||||
output.einfo("Url '%s' blocked by robots.txt" % url)
|
||||
return None
|
||||
|
||||
if not timeout:
|
||||
|
||||
Reference in new issue
Block a user