euscan: don't get robots.txt on ftp

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary 2011-09-06 17:34:50 +02:00
parent a137ef60e3
commit 2210b2610d

View File

@ -217,6 +217,9 @@ def urlallowed(url):
protocol, domain = urlparse.urlparse(url)[:2] protocol, domain = urlparse.urlparse(url)[:2]
if 'protocol' == 'ftp':
return True
baseurl = '%s://%s' % (protocol, domain) baseurl = '%s://%s' % (protocol, domain)
robotsurl = urlparse.urljoin(baseurl, 'robots.txt') robotsurl = urlparse.urljoin(baseurl, 'robots.txt')