diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index c9bf722..ef7f4c8 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -217,6 +217,9 @@ def urlallowed(url): protocol, domain = urlparse.urlparse(url)[:2] + if 'protocol' == 'ftp': + return True + baseurl = '%s://%s' % (protocol, domain) robotsurl = urlparse.urljoin(baseurl, 'robots.txt')