euscan: ignore robots.txt for fedorahosted.org

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-07-16 07:12:09 +02:00
parent 1a7efc58a7
commit 28b913295a

View File

@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
'(.*)chromium.org(.*)', '(.*)chromium.org(.*)',
'(.*)nodejs.org(.*)', '(.*)nodejs.org(.*)',
'(.*)download.mono-project.com(.*)', '(.*)download.mono-project.com(.*)',
'(.*)fedorahosted.org(.*)',
] ]
from out import EuscanOutput from out import EuscanOutput