From 2210b2610d5c1540b99be2f5e8a66b1c5f5d35d9 Mon Sep 17 00:00:00 2001
From: Corentin Chary <corentincj@iksaif.net>
Date: Tue, 6 Sep 2011 17:34:50 +0200
Subject: [PATCH] euscan: don't get robots.txt on ftp

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
---
 pym/euscan/helpers.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py
index c9bf722..ef7f4c8 100644
--- a/pym/euscan/helpers.py
+++ b/pym/euscan/helpers.py
@@ -217,6 +217,9 @@ def urlallowed(url):
 
     protocol, domain = urlparse.urlparse(url)[:2]
 
+    if 'protocol' == 'ftp':
+        return True
+
     baseurl = '%s://%s' % (protocol, domain)
     robotsurl = urlparse.urljoin(baseurl, 'robots.txt')