euscan: ignore robots.txt for fedorahosted.org

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
2012-07-16 07:12:09 +02:00 · 2012-07-16 07:12:09 +02:00 · 28b913295a
commit 28b913295a
parent 1a7efc58a7
1 changed files with 1 additions and 0 deletions
--- a/pym/euscan/init.py
+++ b/pym/euscan/init.py
@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
    '(.*)chromium.org(.*)',
    '(.*)nodejs.org(.*)',
    '(.*)download.mono-project.com(.*)',
+    '(.*)fedorahosted.org(.*)',
 ]

 from out import EuscanOutput