euscan: ignore robots.txt for fedorahosted.org
Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
parent
1a7efc58a7
commit
28b913295a
@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
|
|||||||
'(.*)chromium.org(.*)',
|
'(.*)chromium.org(.*)',
|
||||||
'(.*)nodejs.org(.*)',
|
'(.*)nodejs.org(.*)',
|
||||||
'(.*)download.mono-project.com(.*)',
|
'(.*)download.mono-project.com(.*)',
|
||||||
|
'(.*)fedorahosted.org(.*)',
|
||||||
]
|
]
|
||||||
|
|
||||||
from out import EuscanOutput
|
from out import EuscanOutput
|
||||||
|
Loading…
Reference in New Issue
Block a user