src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

euscan: add new robots.txt exceptions

Browse Source 
Signed-off-by: Corentin Chary <corentincj@iksaif.net>
This commit is contained in:
Corentin Chary 2011-10-08 08:33:03 +02:00
parent d7f655cdde
commit 277fb4ebe6
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pym/euscan/__init__.py
View File

@ -18,7 +18,7 @@ CONFIG = {
'brute-force-recursive': True, 'brute-force-recursive': True,
'brute-force-false-watermark': 50, 'brute-force-false-watermark': 50,
'scan-dir': True, 'scan-dir': True,
'oneshot': False, 'oneshot': True,
'user-agent' : 'escan (http://euscan.iksaif.net)', 'user-agent' : 'escan (http://euscan.iksaif.net)',
'skip-robots-txt' : False, 'skip-robots-txt' : False,
'cache' : False 'cache' : False
@ -60,4 +60,6 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
'(.*)sourceforge(.*)', '(.*)sourceforge(.*)',
'(.*)github.com', '(.*)github.com',
'(.*)berlios(.*)', '(.*)berlios(.*)',
'(.*)qt.nokia.com(.*)',
'(.*)chromium.org(.*)',
] ]