From 28b913295a9d6489c230e9cafdd5944b7ae4a1ad Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Mon, 16 Jul 2012 07:12:09 +0200 Subject: [PATCH] euscan: ignore robots.txt for fedorahosted.org Signed-off-by: Corentin Chary --- pym/euscan/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index 8efdce4..dad6742 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [ '(.*)chromium.org(.*)', '(.*)nodejs.org(.*)', '(.*)download.mono-project.com(.*)', + '(.*)fedorahosted.org(.*)', ] from out import EuscanOutput