euscan: ignore robots.txt for fedorahosted.org
Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
1 parent
1a7efc58a7
commit
28b913295a
1 file changed
+1
@@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
|
||||
'(.*)chromium.org(.*)',
|
||||
'(.*)nodejs.org(.*)',
|
||||
'(.*)download.mono-project.com(.*)',
|
||||
'(.*)fedorahosted.org(.*)',
|
||||
]
|
||||
|
||||
from out import EuscanOutput
|
||||
|
||||
Reference in new issue
Block a user