src_prepare/euscan-ng
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lovely day for PEP8 and pylint!

Browse Source
This commit is contained in:
volpino
2012-04-28 18:16:05 +02:00
parent 4e4f9643ac
commit 8c91855a58
37 changed files with 722 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
pym/euscan/__init__.py
View File

@@ -5,7 +5,6 @@
__version__ = "git"
import sys
from portage.output import EOutput
@@ -19,15 +18,16 @@ CONFIG = {
'brute-force-false-watermark': 50,
'scan-dir': True,
'oneshot': True,
'user-agent' : 'escan (http://euscan.iksaif.net)',
'skip-robots-txt' : False,
'cache' : False
'user-agent': 'escan (http://euscan.iksaif.net)',
'skip-robots-txt': False,
'cache': False
}
output = EOutput(CONFIG['quiet'])
BLACKLIST_VERSIONS = [
# Compatibility package for running binaries linked against a pre gcc 3.4 libstdc++, won't be updated
# Compatibility package for running binaries linked against a
# pre gcc 3.4 libstdc++, won't be updated
'>=sys-libs/libstdc++-v3-3.4',
]
@@ -39,21 +39,24 @@ BLACKLIST_PACKAGES = [
]
SCANDIR_BLACKLIST_URLS = [
'mirror://rubygems/(.*)', # Not browsable
'mirror://gentoo/(.*)' # Directory too big
'mirror://rubygems/(.*)', # Not browsable
'mirror://gentoo/(.*)' # Directory too big
]
BRUTEFORCE_BLACKLIST_PACKAGES = [
'net-zope/plonepopoll' # infinite loop any http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz link will work
# infinite loop any
# http://plone.org/products/plonepopoll/releases/*/plonepopoll-2-6-1.tgz
# link will work
'net-zope/plonepopoll'
]
BRUTEFORCE_BLACKLIST_URLS = [
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop
]
ROBOTS_TXT_BLACKLIST_DOMAINS = [

5
pym/euscan/handlers/__init__.py
View File

@@ -1,6 +1,7 @@
from euscan.handlers import generic, php, pypi, rubygem, kde, cpan
handlers = [ kde, php, pypi, rubygem, cpan, generic ]
handlers = [kde, php, pypi, rubygem, cpan, generic]
def find_best_handler(cpv, url):
for handler in handlers:
@@ -8,12 +9,14 @@ def find_best_handler(cpv, url):
return handler
return None
def scan(cpv, url):
handler = find_best_handler(cpv, url)
if handler:
return handler.scan(cpv, url)
return []
def brute_force(cpv, url):
handler = find_best_handler(cpv, url)
if handler:

21
pym/euscan/handlers/cpan.py
View File

@@ -8,9 +8,11 @@ import euscan
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
def can_handle(cpv, url):
return url.startswith('mirror://cpan/')
def guess_package(cp, url):
match = _cpan_package_name_re.search(url)
@@ -27,6 +29,7 @@ def guess_package(cp, url):
return pkg
def gentoo_mangle_version(up_pv):
pv = ""
@@ -39,17 +42,20 @@ def gentoo_mangle_version(up_pv):
c = up_pv[i]
pv += c
digits += int(c.isdigit())
if c == '.': digits = 0
if c == '.':
digits = 0
else:
pv = up_pv
return helpers.gentoo_mangle_version(pv)
def cpan_trim_version(pv):
pv = re.sub('^[a-zA-Z]+', '', pv)
pv = re.sub('[a-zA-Z]$', '', pv)
return pv
def cpan_mangle_version(pv):
pos = pv.find('.')
if pos < 0:
@@ -59,6 +65,7 @@ def cpan_mangle_version(pv):
up_pv = cpan_trim_version(up_pv)
return up_pv
def cpan_vercmp(cp, a, b):
try:
return float(a) - float(b)
@@ -68,6 +75,7 @@ def cpan_vercmp(cp, a, b):
else:
return 1
def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
pkg = guess_package(cp, url)
@@ -107,15 +115,20 @@ def scan(cpv, url):
if helpers.version_filtered(cp, up_ver, up_pv, cpan_vercmp):
continue
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % \
(version['cpanid'][0], version['cpanid'][0:1], version['cpanid'], version['archive'])
url = 'mirror://cpan/authors/id/%s/%s/%s/%s' % (
version['cpanid'][0],
version['cpanid'][0:1],
version['cpanid'],
version['archive']
)
if url == orig_url:
continue
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

31
pym/euscan/handlers/generic.py
View File

@@ -6,10 +6,12 @@ from BeautifulSoup import BeautifulSoup
import portage
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
from euscan import helpers
import euscan
def scan_html(data, url, pattern):
soup = BeautifulSoup(data)
results = []
@@ -28,6 +30,7 @@ def scan_html(data, url, pattern):
return results
def scan_ftp(data, url, pattern):
buf = StringIO.StringIO(data)
results = []
@@ -40,6 +43,7 @@ def scan_ftp(data, url, pattern):
return results
def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
if not steps:
return []
@@ -91,6 +95,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
return versions
def scan(cpv, url):
for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url):
@@ -107,13 +112,17 @@ def scan(cpv, url):
if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url:
euscan.output.einfo("Version: using %s instead of %s" % (newver, ver))
euscan.output.einfo(
"Version: using %s instead of %s" % (newver, ver)
)
ver = newver
template = helpers.template_from_url(resolved_url, ver)
if '${' not in template:
euscan.output.einfo("Url doesn't seems to depend on version: %s not found in %s"
% (ver, resolved_url))
euscan.output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url)
)
return []
else:
euscan.output.einfo("Scanning: %s" % template)
@@ -121,6 +130,7 @@ def scan(cpv, url):
steps = helpers.generate_scan_paths(template)
return scan_directory_recursive(cp, ver, rev, "", steps, url)
def brute_force(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
@@ -155,8 +165,9 @@ def brute_force(cpv, url):
template = helpers.template_from_url(url, ver)
if '${PV}' not in template:
euscan.output.einfo("Url doesn't seems to depend on full version: %s not found in %s"
% (ver, url))
euscan.output.einfo(
"Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url))
return []
else:
euscan.output.einfo("Brute forcing: %s" % template)
@@ -187,11 +198,14 @@ def brute_force(cpv, url):
result.append([url, version])
if len(result) > CONFIG['brute-force-false-watermark']:
euscan.output.einfo("Broken server detected ! Skipping brute force.")
euscan.output.einfo(
"Broken server detected ! Skipping brute force."
)
return []
if CONFIG["brute-force-recursive"]:
for v in helpers.gen_versions(list(components), CONFIG["brute-force"]):
for v in helpers.gen_versions(list(components),
CONFIG["brute-force"]):
if v not in versions and tuple(v) not in done:
versions.append(v)
@@ -200,5 +214,6 @@ def brute_force(cpv, url):
return result
def can_handle(cpv, url):
return True

4
pym/euscan/handlers/kde.py
View File

@@ -1,10 +1,12 @@
from euscan.handlers import generic
def can_handle(cpv, url):
if url.startswith('mirror://kde/'):
return True
return False
def clean_results(results):
ret = []
@@ -15,6 +17,7 @@ def clean_results(results):
return ret
def scan(cpv, url):
results = generic.scan(cpv, url)
@@ -24,6 +27,7 @@ def scan(cpv, url):
return clean_results(results)
def brute_force(cpv, url):
results = generic.brute_force(cpv, url)

6
pym/euscan/handlers/php.py
View File

@@ -6,6 +6,7 @@ import xml.dom.minidom
from euscan import helpers
import euscan
def can_handle(cpv, url):
if url.startswith('http://pear.php.net/get/'):
return True
@@ -13,6 +14,7 @@ def can_handle(cpv, url):
return True
return False
def guess_package_and_channel(cp, url):
match = re.search('http://(.*)/get/(.*)-(.*).tgz', url)
@@ -24,6 +26,7 @@ def guess_package_and_channel(cp, url):
return pkg, host
def scan(cpv, url):
cp, ver, rev = portage.pkgsplit(cpv)
pkg, channel = guess_package_and_channel(cp, url)
@@ -61,9 +64,10 @@ def scan(cpv, url):
if url == orig_url:
continue
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

10
pym/euscan/handlers/pypi.py
View File

@@ -1,5 +1,4 @@
import xmlrpclib
import pprint
import re
import portage
@@ -7,9 +6,11 @@ import portage
from euscan import helpers
import euscan
def can_handle(cpv, url):
return url.startswith('mirror://pypi/')
def guess_package(cp, url):
match = re.search('mirror://pypi/\w+/(.*)/.*', url)
if match:
@@ -19,10 +20,10 @@ def guess_package(cp, url):
return pkg
def scan(cpv, url):
'http://wiki.python.org/moin/PyPiXmlRpc'
package = guess_package(cpv, url)
euscan.output.einfo("Using PyPi XMLRPC: " + package)
@@ -44,10 +45,11 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv):
continue
urls = client.release_urls(package, up_pv)
urls = " ".join([ infos['url'] for infos in urls ])
ret.append(( urls, pv ))
urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv))
return ret
def brute_force(cpv, url):
return []

9
pym/euscan/handlers/rubygem.py
View File

@@ -6,9 +6,11 @@ import urllib2
from euscan import helpers
import euscan
def can_handle(cpv, url):
return url.startswith('mirror://rubygems/')
def guess_gem(cpv, url):
match = re.search('mirror://rubygems/(.*).gem', url)
if match:
@@ -23,12 +25,14 @@ def guess_gem(cpv, url):
return pkg
def scan(cpv, url):
'http://guides.rubygems.org/rubygems-org-api/#gemversion'
gem = guess_gem(cpv, url)
if not gem:
euscan.output.eerror("Can't guess gem name using %s and %s" % (cpv, url))
euscan.output.eerror("Can't guess gem name using %s and %s" % \
(cpv, url))
return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
@@ -61,9 +65,10 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv):
continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append(( url, pv ))
ret.append((url, pv))
return ret
def brute_force(cpv, url):
return []

45
pym/euscan/helpers.py
View File

@@ -18,6 +18,7 @@ from portage import dep
from euscan import CONFIG, BLACKLIST_VERSIONS, ROBOTS_TXT_BLACKLIST_DOMAINS
import euscan
def htop_vercmp(a, b):
def fixver(v):
if v in ['0.11', '0.12', '0.13']:
@@ -27,12 +28,13 @@ def htop_vercmp(a, b):
return simple_vercmp(fixver(a), fixver(b))
VERSION_CMP_PACKAGE_QUIRKS = {
'sys-process/htop' : htop_vercmp
'sys-process/htop': htop_vercmp
}
_v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
# Stolen from g-pypi
def gentoo_mangle_version(up_pv):
"""Convert PV to MY_PV if needed
@@ -146,6 +148,7 @@ def gentoo_mangle_version(up_pv):
return pv
def cast_int_components(version):
for i, obj in enumerate(version):
try:
@@ -154,6 +157,7 @@ def cast_int_components(version):
pass
return version
def simple_vercmp(a, b):
if a == b:
return 0
@@ -173,11 +177,13 @@ def simple_vercmp(a, b):
else:
return 1
def vercmp(package, a, b):
if package in VERSION_CMP_PACKAGE_QUIRKS:
return VERSION_CMP_PACKAGE_QUIRKS[package](a, b)
return simple_vercmp(a, b)
def version_is_nightly(a, b):
a = pkg_resources.parse_version(a)
b = pkg_resources.parse_version(b)
@@ -188,6 +194,7 @@ def version_is_nightly(a, b):
return True
return False
def version_blacklisted(cp, version):
rule = None
cpv = '%s-%s' % (cp, version)
@@ -205,6 +212,7 @@ def version_blacklisted(cp, version):
euscan.output.einfo("%s is blacklisted by rule %s" % (cpv, bv))
return rule is not None
def version_change_end_sep(version):
match = re.match('.*' + _v_end, version)
if not match:
@@ -218,6 +226,7 @@ def version_change_end_sep(version):
return None
return version.replace(end, newend)
def version_filtered(cp, base, version, vercmp=vercmp):
if vercmp(cp, base, version) >= 0:
return True
@@ -230,6 +239,7 @@ def version_filtered(cp, base, version, vercmp=vercmp):
return False
def generate_templates_vars(version):
ret = []
@@ -246,6 +256,7 @@ def generate_templates_vars(version):
ret.reverse()
return ret
def template_from_url(url, version):
prefix, chunks = url.split('://')
chunks = chunks.split('/')
@@ -261,6 +272,7 @@ def template_from_url(url, version):
return prefix + "://" + "/".join(chunks)
def url_from_template(url, version):
components = split_version(version)
@@ -270,6 +282,7 @@ def url_from_template(url, version):
return url
# Stolen from distutils.LooseVersion
# Used for brute force to increment the version
def split_version(version):
@@ -282,6 +295,7 @@ def split_version(version):
pass
return components
def join_version(components):
version = ""
for i in range(len(components)):
@@ -292,6 +306,7 @@ def join_version(components):
version += "."
return version
def increment_version(components, level):
n = len(components)
@@ -307,6 +322,7 @@ def increment_version(components, level):
return components
def gen_versions(components, level):
n = len(components)
depth = level
@@ -325,6 +341,7 @@ def gen_versions(components, level):
return versions
def timeout_for_url(url):
if 'sourceforge' in url:
timeout = 15
@@ -332,13 +349,16 @@ def timeout_for_url(url):
timeout = 5
return timeout
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
""" RobotParser cache """
rpcache = {}
def urlallowed(url):
if CONFIG['skip-robots-txt']:
return True
@@ -359,7 +379,7 @@ def urlallowed(url):
baseurl = '%s://%s' % (protocol, domain)
robotsurl = urlparse.urljoin(baseurl, 'robots.txt')
if rpcache.has_key(baseurl):
if baseurl in rpcache:
rp = rpcache[baseurl]
else:
from socket import setdefaulttimeout, getdefaulttimeout
@@ -379,6 +399,7 @@ def urlallowed(url):
return rp.can_fetch(CONFIG['user-agent'], url) if rp else False
def urlopen(url, timeout=None, verb="GET"):
if not urlallowed(url):
euscan.output.einfo("Url '%s' blocked by robots.txt" % url)
@@ -410,6 +431,7 @@ def urlopen(url, timeout=None, verb="GET"):
return opener.open(request, None, timeout)
def tryurl(fileurl, template):
result = True
@@ -429,13 +451,16 @@ def tryurl(fileurl, template):
headers = fp.info()
if 'Content-disposition' in headers and basename not in headers['Content-disposition']:
if 'Content-disposition' in headers and \
basename not in headers['Content-disposition']:
result = None
elif 'Content-Length' in headers and headers['Content-Length'] == '0':
result = None
elif 'Content-Type' in headers and 'text/html' in headers['Content-Type']:
elif 'Content-Type' in headers and \
'text/html' in headers['Content-Type']:
result = None
elif 'Content-Type' in headers and 'application/x-httpd-php' in headers['Content-Type']:
elif 'Content-Type' in headers and \
'application/x-httpd-php' in headers['Content-Type']:
result = None
elif fp.geturl() != fileurl:
regex = regex_from_template(template)
@@ -443,10 +468,10 @@ def tryurl(fileurl, template):
basename2 = os.path.basename(fp.geturl())
# Redirect to another (earlier?) version
if basename != basename2 and (re.match(regex, fp.geturl()) or re.match(baseregex, basename2)):
if basename != basename2 and (re.match(regex, fp.geturl()) or \
re.match(baseregex, basename2)):
result = None
if result:
result = (fp.geturl(), fp.info())
@@ -459,6 +484,7 @@ def tryurl(fileurl, template):
return result
def regex_from_template(template):
# Escape
template = re.escape(template)
@@ -483,6 +509,7 @@ def regex_from_template(template):
template = template + r'/?$'
return template
def basedir_from_template(template):
idx = template.find('${')
if idx == -1:
@@ -494,6 +521,7 @@ def basedir_from_template(template):
return template[0:idx]
def generate_scan_paths(url):
prefix, chunks = url.split('://')
chunks = chunks.split('/')
@@ -511,6 +539,7 @@ def generate_scan_paths(url):
return steps
def parse_mirror(uri):
from random import shuffle
@@ -526,7 +555,7 @@ def parse_mirror(uri):
return None
mirrorname = uri[9:eidx]
path = uri[eidx+1:]
path = uri[eidx + 1:]
if mirrorname in mirrors:
mirrors = mirrors[mirrorname]

26
pym/euscan/scan.py
View File

@@ -4,10 +4,8 @@ import sys
import portage
from portage.dbapi import porttree
from portage.output import white, yellow, turquoise, green, teal, red, EOutput
import gentoolkit.pprinter as pp
from gentoolkit import errors
from gentoolkit.query import Query
from gentoolkit.eclean.search import (port_settings)
@@ -17,6 +15,7 @@ from euscan import helpers
import euscan
def filter_versions(cp, versions):
filtered = {}
@@ -32,7 +31,8 @@ def filter_versions(cp, versions):
filtered[version] = url
return [ (cp, filtered[version], version) for version in filtered ]
return [(cp, filtered[version], version) for version in filtered]
def scan_upstream_urls(cpv, urls):
versions = []
@@ -72,7 +72,9 @@ def scan_upstream(query):
)
if not matches:
sys.stderr.write(pp.warn("No package matching '%s'" % pp.pkgquery(query)))
sys.stderr.write(
pp.warn("No package matching '%s'" % pp.pkgquery(query))
)
return []
matches = sorted(matches)
@@ -87,11 +89,15 @@ def scan_upstream(query):
return []
if pkg.cp in BLACKLIST_PACKAGES:
sys.stderr.write(pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)))
sys.stderr.write(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
)
return []
if not CONFIG['quiet']:
pp.uprint(" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())))
pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
)
pp.uprint()
ebuild_path = pkg.ebuild_path()
@@ -104,8 +110,8 @@ def scan_upstream(query):
cpv = pkg.cpv
metadata = {
"EAPI" : port_settings["EAPI"],
"SRC_URI" : pkg.environment("SRC_URI", False),
"EAPI": port_settings["EAPI"],
"SRC_URI": pkg.environment("SRC_URI", False),
}
use = frozenset(port_settings["PORTAGE_USE"].split())
try:
@@ -113,7 +119,9 @@ def scan_upstream(query):
aalist = porttree._parse_uri_map(cpv, metadata)
except Exception as e:
sys.stderr.write(pp.warn("%s\n" % str(e)))
sys.stderr.write(pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)))
sys.stderr.write(
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
)
return []
if "mirror" in portage.settings.features: