src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lovely day for PEP8 and pylint!

Browse Source
This commit is contained in:
volpino
2012-04-28 18:16:05 +02:00
parent 4e4f9643ac
commit 8c91855a58
37 changed files with 722 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pym/euscan/handlers/__init__.py
View File

@ -1,6 +1,7 @@
from euscan.handlers import generic, php, pypi, rubygem, kde, cpan
handlers = [ kde, php, pypi, rubygem, cpan, generic ]
handlers = [kde, php, pypi, rubygem, cpan, generic]
def find_best_handler(cpv, url):
for handler in handlers:
@ -8,12 +9,14 @@ def find_best_handler(cpv, url):
return handler
return None
def scan(cpv, url):
handler = find_best_handler(cpv, url)
if handler:
return handler.scan(cpv, url)
return []
def brute_force(cpv, url):
handler = find_best_handler(cpv, url)
if handler:

21
pym/euscan/handlers/cpan.py
View File

@ -8,9 +8,11 @@ import euscan
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
def can_handle(cpv, url):
return url.startswith('mirror://cpan/')
def guess_package(cp, url):
match = _cpan_package_name_re.search(url)
@ -27,6 +29,7 @@ def guess_package(cp, url):
return pkg
def gentoo_mangle_version(up_pv):
pv = ""
@ -39,17 +42,20 @@ def gentoo_mangle_version(up_pv):
c = up_pv[i]
pv += c
digits += int(c.isdigit())
if c == '.': digits = 0
if c == '.':
digits = 0
else:
pv = up_pv
return helpers.gentoo_mangle_version(pv)
def cpan_trim_version(pv):
pv = re.sub('^[a-zA-Z]+', '', pv)
pv = re.sub('[a-zA-Z]$', '', pv)
return pv
def cpan_mangle_version(pv):
pos = pv.find('.')
if pos < 0:
@ -59,6 +65,7 @@ def cpan_mangle_version(pv):
up_pv = cpan_trim_version(up_pv)
return up_pv
def cpan_vercmp(cp, a, b):
try:
return float(a) - float(b)
@ -68,6 +75,7 @@ def cpan_vercmp(cp, a, b):
else:
return 1
def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
pkg = guess_package(cp, url)
@ -107,15 +115,20 @@ def scan(cpv, url):
if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp):
continue
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % \
(version['cpanid'][0], version['cpanid'][0:1], version['cpanid'], version['archive'])
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
version['cpanid'][0],
version['cpanid'][0:1],
version['cpanid'],
version['archive']
)
if url == orig_url:
continue
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

31
pym/euscan/handlers/generic.py
View File

@ -6,10 +6,12 @@ from BeautifulSoup import BeautifulSoup
import portage
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import helpers
import euscan
def scan_html(data, url, pattern):
soup = BeautifulSoup(data)
results = []
@ -28,6 +30,7 @@ def scan_html(data, url, pattern):
return results
def scan_ftp(data, url, pattern):
buf = StringIO.StringIO(data)
results = []
@ -40,6 +43,7 @@ def scan_ftp(data, url, pattern):
return results
def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
if not steps:
return []
@ -91,6 +95,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
return versions
def scan(cpv, url):
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
@ -107,13 +112,17 @@ def scan(cpv, url):
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
euscan.output.einfo("Version: using %s instead of %s" % (newver, ver))
euscan.output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
ver = newver
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
euscan.output.einfo("Url doesn't seems to depend on version: %s not found in %s"
% (ver, resolved_url))
euscan.output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
return []
else:
euscan.output.einfo("Scanning: %s" % template)
@ -121,6 +130,7 @@ def scan(cpv, url):
steps = helpers.generate_scan_paths(template)
return scan_directory_recursive(cp, ver, rev, "", steps, url)
def brute_force(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
@ -155,8 +165,9 @@ def brute_force(cpv, url):
template = helpers.template_from_url(url, ver)
if '${PV}' not in template:
euscan.output.einfo("Url doesn't seems to depend on full version: %s not found in %s"
% (ver, url))
euscan.output.einfo(
"Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url))
return []
else:
euscan.output.einfo("Brute forcing: %s" % template)
@ -187,11 +198,14 @@ def brute_force(cpv, url):
result.append([url, version])
if len(result) > CONFIG['brute-force-false-watermark']:
euscan.output.einfo("Broken server detected ! Skipping brute force.")
euscan.output.einfo(
"Broken server detected ! Skipping brute force."
)
return []
if CONFIG["brute-force-recursive"]:
for v in helpers.gen_versions(list(components), CONFIG["brute-force"]):
for v in helpers.gen_versions(list(components),
CONFIG["brute-force"]):
if v not in versions and tuple(v) not in done:
versions.append(v)
@ -200,5 +214,6 @@ def brute_force(cpv, url):
return result
def can_handle(cpv, url):
return True

4
pym/euscan/handlers/kde.py
View File

@ -1,10 +1,12 @@
from euscan.handlers import generic
def can_handle(cpv, url):
if url.startswith('mirror://kde/'):
return True
return False
def clean_results(results):
ret = []
@ -15,6 +17,7 @@ def clean_results(results):
return ret
def scan(cpv, url):
results = generic.scan(cpv, url)
@ -24,6 +27,7 @@ def scan(cpv, url):
return clean_results(results)
def brute_force(cpv, url):
results = generic.brute_force(cpv, url)

6
pym/euscan/handlers/php.py
View File

@ -6,6 +6,7 @@ import xml.dom.minidom
from euscan import helpers
import euscan
def can_handle(cpv, url):
if url.startswith('http://pear.php.net/get/'):
return True
@ -13,6 +14,7 @@ def can_handle(cpv, url):
return True
return False
def guess_package_and_channel(cp, url):
match = re.search('http://(.*)/get/(.*)-(.*).tgz', url)
@ -24,6 +26,7 @@ def guess_package_and_channel(cp, url):
return pkg, host
def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
pkg, channel = guess_package_and_channel(cp, url)
@ -61,9 +64,10 @@ def scan(cpv, url):
if url == orig_url:
continue
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

10
pym/euscan/handlers/pypi.py
View File

@ -1,5 +1,4 @@
import xmlrpclib
import pprint
import re
import portage
@ -7,9 +6,11 @@ import portage
from euscan import helpers
import euscan
def can_handle(cpv, url):
return url.startswith('mirror://pypi/')
def guess_package(cp, url):
match = re.search('mirror://pypi/\w+/(.*)/.*', url)
if match:
@ -19,10 +20,10 @@ def guess_package(cp, url):
return pkg
def scan(cpv, url):
'http://wiki.python.org/moin/PyPiXmlRpc'
package = guess_package(cpv, url)
euscan.output.einfo("Using PyPi XMLRPC: " + package)
@ -44,10 +45,11 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv):
continue
urls = client.release_urls(package, up_pv)
urls = " ".join([ infos['url'] for infos in urls ])
ret.append(( urls, pv ))
urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv))
return ret
def brute_force(cpv, url):
return []

9
pym/euscan/handlers/rubygem.py
View File

@ -6,9 +6,11 @@ import urllib2
from euscan import helpers
import euscan
def can_handle(cpv, url):
return url.startswith('mirror://rubygems/')
def guess_gem(cpv, url):
match = re.search('mirror://rubygems/(.*).gem', url)
if match:
@ -23,12 +25,14 @@ def guess_gem(cpv, url):
return pkg
def scan(cpv, url):
'http://guides.rubygems.org/rubygems-org-api/#gemversion'
gem = guess_gem(cpv, url)
if not gem:
euscan.output.eerror("Can't guess gem name using %s and %s" % (cpv, url))
euscan.output.eerror("Can't guess gem name using %s and %s" % \
(cpv, url))
return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
@ -61,9 +65,10 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv):
continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []