Merge git://github.com/volpino/euscan

Conflicts:
	pym/euscan/scan.py
This commit is contained in:
Corentin Chary 2012-05-24 23:47:55 +02:00
commit 687851ffcb
10 changed files with 332 additions and 161 deletions

View File

@ -6,36 +6,43 @@ Distributed under the terms of the GNU General Public License v2
from __future__ import print_function from __future__ import print_function
""" Meta """
# Meta
__author__ = "Corentin Chary (iksaif)" __author__ = "Corentin Chary (iksaif)"
__email__ = "corentin.chary@gmail.com" __email__ = "corentin.chary@gmail.com"
__version__ = "git" __version__ = "git"
__productname__ = "euscan" __productname__ = "euscan"
__description__ = "A tool to detect new upstream releases." __description__ = "A tool to detect new upstream releases."
__version__ = "git"
""" Imports """
# Imports
import sys import sys
import getopt import getopt
import errno import errno
import httplib import httplib
from portage.output import white, yellow, turquoise, green, EOutput from portage.output import white, yellow, turquoise, green
from portage.exception import AmbiguousPackageName from portage.exception import AmbiguousPackageName
from gentoolkit import pprinter as pp from gentoolkit import pprinter as pp
from gentoolkit.eclean.search import (port_settings) from gentoolkit.eclean.search import (port_settings)
from gentoolkit.errors import GentoolkitException from gentoolkit.errors import GentoolkitException
import euscan from euscan import CONFIG, output
from euscan import CONFIG
from euscan.scan import scan_upstream from euscan.scan import scan_upstream
""" Globals """
# Globals
def exit_helper(status):
if CONFIG["format"]:
print(output.get_formatted_output())
sys.exit(status)
def setupSignals(): def setup_signals():
"""This block ensures that ^C interrupts are handled quietly.""" """This block ensures that ^C interrupts are handled quietly."""
import signal import signal
@ -43,14 +50,14 @@ def setupSignals():
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN)
print() print()
sys.exit(errno.EINTR) exit_helper(errno.EINTR)
signal.signal(signal.SIGINT, exithandler) signal.signal(signal.SIGINT, exithandler)
signal.signal(signal.SIGTERM, exithandler) signal.signal(signal.SIGTERM, exithandler)
signal.signal(signal.SIGPIPE, signal.SIG_DFL) signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def printVersion(): def print_version():
"""Output the version info.""" """Output the version info."""
print("%s (%s) - %s" \ print("%s (%s) - %s" \
% (__productname__, __version__, __description__)) % (__productname__, __version__, __description__))
@ -60,23 +67,26 @@ def printVersion():
print("Distributed under the terms of the GNU General Public License v2") print("Distributed under the terms of the GNU General Public License v2")
def printUsage(_error=None, help=None): def print_usage(_error=None, help=None):
"""Print help message. May also print partial help to stderr if an """Print help message. May also print partial help to stderr if an
error from {'options'} is specified.""" error from {'options'} is specified."""
out = sys.stdout out = sys.stdout
if _error: if _error:
out = sys.stderr out = sys.stderr
if not _error in ('global-options', 'packages',): if not _error in ('global-options', 'packages',):
_error = None _error = None
if not _error and not help: if not _error and not help:
help = 'all' help = 'all'
if _error in ('global-options',): if _error in ('global-options',):
print(pp.error("Wrong option on command line."), file=out) output.eerror("Wrong option on command line.\n")
print(file=out)
if _error in ('packages',): if _error in ('packages',):
print(pp.error("You need to specify exactly one package."), file=out) output.eerror("You need to specify exactly one package.\n")
print(file=out)
print(white("Usage:"), file=out) print(white("Usage:"), file=out)
if _error in ('global-options', 'packages',) or help == 'all': if _error in ('global-options', 'packages',) or help == 'all':
print(" " + turquoise(__productname__), print(" " + turquoise(__productname__),
@ -106,14 +116,19 @@ def printUsage(_error=None, help=None):
" (default: 2)\n" + " (default: 2)\n" +
" " * 29 + "bigger levels will generate more versions numbers\n" + " " * 29 + "bigger levels will generate more versions numbers\n" +
" " * 29 + "0 means disabled", file=out) " " * 29 + "0 means disabled", file=out)
print(yellow(" -f, --format=<format>") +
" - define the output " + yellow("<format>") +
" (available: json)", file=out)
print(file=out) print(file=out)
if _error in ('packages',) or help: if _error in ('packages',) or help:
print(green(" package") + print(green(" package") +
" - the packages (or ebuilds) you want to scan", " - the packages (or ebuilds) you want to scan",
file=out) file=out)
print(file=out) print(file=out)
'''print( "More detailed instruction can be found in",
turquoise("`man %s`" % __productname__), file=out)''' #print( "More detailed instruction can be found in",
#turquoise("`man %s`" % __productname__), file=out)
class ParseArgsException(Exception): class ParseArgsException(Exception):
@ -125,12 +140,12 @@ class ParseArgsException(Exception):
return repr(self.value) return repr(self.value)
def parseArgs(): def parse_args():
"""Parse the command line arguments. Raise exceptions on """Parse the command line arguments. Raise exceptions on
errors. Returns package and affect the CONFIG dict. errors. Returns packages and affects the CONFIG dict.
""" """
def optionSwitch(opts): def option_switch(opts):
"""local function for interpreting command line options """local function for interpreting command line options
and setting options accordingly""" and setting options accordingly"""
return_code = True return_code = True
@ -151,29 +166,35 @@ def parseArgs():
CONFIG['brute-force'] = int(a) CONFIG['brute-force'] = int(a)
elif o in ("-v", "--verbose") and not CONFIG['quiet']: elif o in ("-v", "--verbose") and not CONFIG['quiet']:
CONFIG['verbose'] += 1 CONFIG['verbose'] += 1
elif o in ("-f", "--format"):
CONFIG['format'] = a
CONFIG['nocolor'] = True
pp.output.nocolor()
else: else:
return_code = False return_code = False
return return_code return return_code
' here are the different allowed command line options (getopt args) ' # here are the different allowed command line options (getopt args)
getopt_options = {'short': {}, 'long': {}} getopt_options = {'short': {}, 'long': {}}
getopt_options['short']['global'] = "hVCqv1b:" getopt_options['short']['global'] = "hVCqv1bf:"
getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet", getopt_options['long']['global'] = [
"verbose", "oneshot", "brute-force="] "help", "version", "nocolor", "quiet", "verbose", "oneshot",
"brute-force=", "format="
]
short_opts = getopt_options['short']['global'] short_opts = getopt_options['short']['global']
long_opts = getopt_options['long']['global'] long_opts = getopt_options['long']['global']
opts_mode = 'global' opts_mode = 'global'
' apply getopts to command line, show partial help on failure ' # apply getopts to command line, show partial help on failure
try: try:
opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
except: except:
raise ParseArgsException(opts_mode + '-options') raise ParseArgsException(opts_mode + '-options')
' set options accordingly ' # set options accordingly
optionSwitch(opts) option_switch(opts)
if len(args) < 1: if len(args) < 1:
raise ParseArgsException('packages') raise ParseArgsException('packages')
@ -183,81 +204,80 @@ def parseArgs():
def main(): def main():
"""Parse command line and execute all actions.""" """Parse command line and execute all actions."""
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true') CONFIG['nocolor'] = (
or not sys.stdout.isatty()) port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()
)
if CONFIG['nocolor']: if CONFIG['nocolor']:
pp.output.nocolor() pp.output.nocolor()
' parse command line options and actions '
# parse command line options and actions
try: try:
packages = parseArgs() queries = parse_args()
except ParseArgsException as e: except ParseArgsException as e:
if e.value == 'help': if e.value == 'help':
printUsage(help='all') print_usage(help='all')
sys.exit(0) exit_helper(0)
elif e.value[:5] == 'help-':
printUsage(help=e.value[5:]) elif e.value[:5] == 'help-':
sys.exit(0) print_usage(help=e.value[5:])
elif e.value == 'version': exit_helper(0)
printVersion()
sys.exit(0) elif e.value == 'version':
else: print_version()
printUsage(e.value) exit_helper(0)
sys.exit(errno.EINVAL)
else:
print_usage(e.value)
exit_helper(errno.EINVAL)
""" Change euscan's output """
euscan.output = EOutput(CONFIG['quiet'])
if CONFIG['verbose'] > 2: if CONFIG['verbose'] > 2:
httplib.HTTPConnection.debuglevel = 1 httplib.HTTPConnection.debuglevel = 1
for package in packages: for query in queries:
ret = [] ret = []
output.set_query(query)
try: try:
ret = scan_upstream(package) ret = scan_upstream(query)
except AmbiguousPackageName as e: except AmbiguousPackageName as e:
pkgs = e.args[0] pkgs = e.args[0]
for candidate in pkgs: output.eerror("\n".join(pkgs))
print(candidate)
from os.path import basename # To get the short name from os.path import basename # To get the short name
print(file=sys.stderr) output.error(
print( "The short ebuild name '%s' is ambiguous. Please specify" %
pp.error( basename(pkgs[0]),
"The short ebuild name '%s' is ambiguous. Please specify" \ "one of the above fully-qualified ebuild names instead."
% basename(pkgs[0])
),
file=sys.stderr, end=""
) )
pp.die(1, "one of the above fully-qualified ebuild names instead.") exit_helper(1)
except GentoolkitException as err:
pp.die(1, '%s: %s' % (package, str(err)))
except Exception as err:
pp.die(1, '%s: %s' % (package, str(err)))
if not CONFIG['quiet']: except GentoolkitException as err:
output.eerror('%s: %s' % (query, str(err)))
exit_helper(1)
except Exception as err:
output.eerror('%s: %s' % (query, str(err)))
exit_helper(1)
if not CONFIG['quiet'] and not CONFIG['format']:
print() print()
for cp, url, version in ret: if ret is not None:
if not CONFIG['quiet']: if len(ret) > 0:
print("Upstream Version: " for cp, url, version, handler, confidence in ret:
+ pp.number("%s" % version) output.result(cp, version, url, handler, confidence)
+ pp.path(" %s" % url)) elif not CONFIG['quiet']:
else: output.ewarn(
print(pp.cpv("%s-%s" % (cp, version)) "Didn't find any new version, check package's homepage " +
+ ": " + pp.path(url)) "for more informations"
)
if not len(ret) and not CONFIG['quiet']: output.set_query(None)
print(pp.warn("Didn't find any new version, "
+ "check package's homepage for "
+ "more informations"))
if __name__ == "__main__": if __name__ == "__main__":
try: setup_signals()
setupSignals()
main() main()
except KeyboardInterrupt: exit_helper(0)
print("Aborted.")
sys.exit(errno.EINTR)
sys.exit(0)

View File

@ -3,11 +3,17 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com> # Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Distributed under the terms of the GNU General Public License v2 # Distributed under the terms of the GNU General Public License v2
from io import StringIO
from collections import defaultdict
import json
from gentoolkit import pprinter as pp
from portage.output import EOutput
__version__ = "git" __version__ = "git"
from portage.output import EOutput
CONFIG = { CONFIG = {
'nocolor': False, 'nocolor': False,
'quiet': False, 'quiet': False,
@ -20,11 +26,11 @@ CONFIG = {
'oneshot': True, 'oneshot': True,
'user-agent': 'escan (http://euscan.iksaif.net)', 'user-agent': 'escan (http://euscan.iksaif.net)',
'skip-robots-txt': False, 'skip-robots-txt': False,
'cache': False 'cache': False,
'format': None,
'indent': 2
} }
output = EOutput(CONFIG['quiet'])
BLACKLIST_VERSIONS = [ BLACKLIST_VERSIONS = [
# Compatibility package for running binaries linked against a # Compatibility package for running binaries linked against a
# pre gcc 3.4 libstdc++, won't be updated # pre gcc 3.4 libstdc++, won't be updated
@ -53,10 +59,11 @@ BRUTEFORCE_BLACKLIST_PACKAGES = [
BRUTEFORCE_BLACKLIST_URLS = [ BRUTEFORCE_BLACKLIST_URLS = [
'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop 'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop
'http://hydra.nixos.org/build/(.*)', # infinite loop 'http://hydra.nixos.org/build/(.*)', # infinite loop
'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop # Doesn't respect 404, infinite loop
'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop 'http://www.rennings.net/gentoo/distfiles/(.*)',
'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop 'http://art.gnome.org/download/(.*)',
'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop 'http://barelysufficient.org/~olemarkus/(.*)',
'http://olemarkus.org/~olemarkus/(.*)',
] ]
ROBOTS_TXT_BLACKLIST_DOMAINS = [ ROBOTS_TXT_BLACKLIST_DOMAINS = [
@ -67,3 +74,94 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
'(.*)chromium.org(.*)', '(.*)chromium.org(.*)',
'(.*)nodejs.org(.*)', '(.*)nodejs.org(.*)',
] ]
class EOutputFile(EOutput):
"""
Override of EOutput, allows to specify an output file for writes
"""
def __init__(self, out_file=None, *args, **kwargs):
super(EOutputFile, self).__init__(*args, **kwargs)
self.out_file = out_file
def _write(self, f, msg):
if self.out_file is None:
super(EOutputFile, self)._write(f, msg)
else:
super(EOutputFile, self)._write(self.out_file, msg)
class EuscanOutput(object):
"""
Class that handles output for euscan
"""
def __init__(self, config):
self.config = config
self.queries = defaultdict(dict)
self.current_query = None
def set_query(self, query):
self.current_query = query
if query is not None:
if not query in self.queries:
self.queries[query] = {
"messages": defaultdict(StringIO),
"result": [],
"metadata": {},
}
def get_formatted_output(self):
data = {}
for query in self.queries:
data[query] = {
"result": self.queries[query]["result"],
"metadata": self.queries[query]["metadata"],
"messages": {}
}
for key in self.queries[query]["messages"]:
if key not in ("ebegin", "eend"):
_msg = self.queries[query]["messages"][key].getvalue()
val = [x for x in _msg.split("\n") if x]
data[query]["messages"][key] = val
if self.config["format"].lower() == "json":
return json.dumps(data, indent=self.config["indent"])
else:
raise TypeError("Invalid output format")
def result(self, cp, version, url, handler, confidence):
from euscan.helpers import get_version_type
if self.config['format']:
_curr = self.queries[self.current_query]
_curr["result"].append(
{"version": version, "urls": [url], "handler": handler,
"confidence": confidence, "type": get_version_type(version)}
)
else:
if not self.config['quiet']:
print "Upstream Version:", pp.number("%s" % version),
print pp.path(" %s" % url)
else:
print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url)
def metadata(self, key, value, show=True):
if self.config["format"]:
self.queries[self.current_query]["metadata"][key] = value
elif show:
print "%s: %s" % (key.capitalize(), value)
def __getattr__(self, key):
if self.config["format"]:
out_file = self.queries[self.current_query]["messages"][key]
_output = EOutputFile(out_file=out_file,
quiet=self.config['quiet'])
ret = getattr(_output, key)
else:
ret = getattr(EOutputFile(quiet=self.config['quiet']), key)
return ret
output = EuscanOutput(CONFIG)

View File

@ -3,8 +3,10 @@ import portage
import urllib2 import urllib2
import json import json
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "cpan"
CONFIDENCE = 100.0
_cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
@ -83,7 +85,7 @@ def scan(cpv, url):
orig_url = url orig_url = url
url = 'http://search.cpan.org/api/dist/%s' % pkg url = 'http://search.cpan.org/api/dist/%s' % pkg
euscan.output.einfo("Using: " + url) output.einfo("Using: " + url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@ -125,7 +127,7 @@ def scan(cpv, url):
if url == orig_url: if url == orig_url:
continue continue
ret.append((url, pv)) ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret

View File

@ -7,9 +7,13 @@ from BeautifulSoup import BeautifulSoup
import portage import portage
from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \ from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers
from euscan import helpers
import euscan HANDLER_NAME = "generic"
CONFIDENCE = 50.0
BRUTEFORCE_HANDLER_NAME = "brute_force"
BRUTEFORCE_CONFIDENCE = 30.0
def scan_html(data, url, pattern): def scan_html(data, url, pattern):
@ -53,7 +57,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
steps = steps[1:] steps = steps[1:]
euscan.output.einfo("Scanning: %s" % url) output.einfo("Scanning: %s" % url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@ -87,7 +91,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
path = url + path path = url + path
if not steps and path not in orig_url: if not steps and path not in orig_url:
versions.append((path, pv)) versions.append((path, pv, HANDLER_NAME, CONFIDENCE))
if steps: if steps:
ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url) ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url)
@ -99,7 +103,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
def scan(cpv, url): def scan(cpv, url):
for bu in SCANDIR_BLACKLIST_URLS: for bu in SCANDIR_BLACKLIST_URLS:
if re.match(bu, url): if re.match(bu, url):
euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu)) output.einfo("%s is blacklisted by rule %s" % (url, bu))
return [] return []
resolved_url = helpers.parse_mirror(url) resolved_url = helpers.parse_mirror(url)
@ -112,23 +116,25 @@ def scan(cpv, url):
if ver not in resolved_url: if ver not in resolved_url:
newver = helpers.version_change_end_sep(ver) newver = helpers.version_change_end_sep(ver)
if newver and newver in resolved_url: if newver and newver in resolved_url:
euscan.output.einfo( output.einfo(
"Version: using %s instead of %s" % (newver, ver) "Version: using %s instead of %s" % (newver, ver)
) )
ver = newver ver = newver
template = helpers.template_from_url(resolved_url, ver) template = helpers.template_from_url(resolved_url, ver)
if '${' not in template: if '${' not in template:
euscan.output.einfo( output.einfo(
"Url doesn't seems to depend on version: %s not found in %s" % "Url doesn't seems to depend on version: %s not found in %s" %
(ver, resolved_url) (ver, resolved_url)
) )
return [] return []
else: else:
euscan.output.einfo("Scanning: %s" % template) output.einfo("Scanning: %s" % template)
steps = helpers.generate_scan_paths(template) steps = helpers.generate_scan_paths(template)
return scan_directory_recursive(cp, ver, rev, "", steps, url) ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
return ret
def brute_force(cpv, url): def brute_force(cpv, url):
@ -140,37 +146,37 @@ def brute_force(cpv, url):
for bp in BRUTEFORCE_BLACKLIST_PACKAGES: for bp in BRUTEFORCE_BLACKLIST_PACKAGES:
if re.match(bp, cp): if re.match(bp, cp):
euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp)) output.einfo("%s is blacklisted by rule %s" % (cp, bp))
return [] return []
for bp in BRUTEFORCE_BLACKLIST_URLS: for bp in BRUTEFORCE_BLACKLIST_URLS:
if re.match(bp, url): if re.match(bp, url):
euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp)) output.einfo("%s is blacklisted by rule %s" % (cp, bp))
return [] return []
euscan.output.einfo("Generating version from " + ver) output.einfo("Generating version from " + ver)
components = helpers.split_version(ver) components = helpers.split_version(ver)
versions = helpers.gen_versions(components, CONFIG["brute-force"]) versions = helpers.gen_versions(components, CONFIG["brute-force"])
""" Remove unwanted versions """ # Remove unwanted versions
for v in versions: for v in versions:
if helpers.vercmp(cp, ver, helpers.join_version(v)) >= 0: if helpers.vercmp(cp, ver, helpers.join_version(v)) >= 0:
versions.remove(v) versions.remove(v)
if not versions: if not versions:
euscan.output.einfo("Can't generate new versions from " + ver) output.einfo("Can't generate new versions from " + ver)
return [] return []
template = helpers.template_from_url(url, ver) template = helpers.template_from_url(url, ver)
if '${PV}' not in template: if '${PV}' not in template:
euscan.output.einfo( output.einfo(
"Url doesn't seems to depend on full version: %s not found in %s" % "Url doesn't seems to depend on full version: %s not found in %s" %
(ver, url)) (ver, url))
return [] return []
else: else:
euscan.output.einfo("Brute forcing: %s" % template) output.einfo("Brute forcing: %s" % template)
result = [] result = []
@ -195,10 +201,11 @@ def brute_force(cpv, url):
if not infos: if not infos:
continue continue
result.append([url, version]) result.append([url, version, BRUTEFORCE_HANDLER_NAME,
BRUTEFORCE_CONFIDENCE])
if len(result) > CONFIG['brute-force-false-watermark']: if len(result) > CONFIG['brute-force-false-watermark']:
euscan.output.einfo( output.einfo(
"Broken server detected ! Skipping brute force." "Broken server detected ! Skipping brute force."
) )
return [] return []

View File

@ -1,5 +1,7 @@
from euscan.handlers import generic from euscan.handlers import generic
HANDLER_NAME = "kde"
def can_handle(cpv, url): def can_handle(cpv, url):
if url.startswith('mirror://kde/'): if url.startswith('mirror://kde/'):
@ -10,10 +12,10 @@ def can_handle(cpv, url):
def clean_results(results): def clean_results(results):
ret = [] ret = []
for path, version in results: for path, version, confidence in results:
if version == '5SUMS': if version == '5SUMS':
continue continue
ret.append((path, version)) ret.append((path, version, HANDLER_NAME, confidence))
return ret return ret

View File

@ -3,8 +3,10 @@ import portage
import urllib2 import urllib2
import xml.dom.minidom import xml.dom.minidom
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "php"
CONFIDENCE = 100.0
def can_handle(cpv, url): def can_handle(cpv, url):
@ -34,7 +36,7 @@ def scan(cpv, url):
orig_url = url orig_url = url
url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower()) url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower())
euscan.output.einfo("Using: " + url) output.einfo("Using: " + url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@ -64,7 +66,7 @@ def scan(cpv, url):
if url == orig_url: if url == orig_url:
continue continue
ret.append((url, pv)) ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret

View File

@ -3,8 +3,10 @@ import re
import portage import portage
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "pypi"
CONFIDENCE = 100.0
def can_handle(cpv, url): def can_handle(cpv, url):
@ -26,7 +28,7 @@ def scan(cpv, url):
package = guess_package(cpv, url) package = guess_package(cpv, url)
euscan.output.einfo("Using PyPi XMLRPC: " + package) output.einfo("Using PyPi XMLRPC: " + package)
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
versions = client.package_releases(package) versions = client.package_releases(package)
@ -46,7 +48,7 @@ def scan(cpv, url):
continue continue
urls = client.release_urls(package, up_pv) urls = client.release_urls(package, up_pv)
urls = " ".join([infos['url'] for infos in urls]) urls = " ".join([infos['url'] for infos in urls])
ret.append((urls, pv)) ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret

View File

@ -3,8 +3,10 @@ import portage
import json import json
import urllib2 import urllib2
from euscan import helpers from euscan import helpers, output
import euscan
HANDLER_NAME = "rubygem"
CONFIDENCE = 100.0
def can_handle(cpv, url): def can_handle(cpv, url):
@ -31,13 +33,13 @@ def scan(cpv, url):
gem = guess_gem(cpv, url) gem = guess_gem(cpv, url)
if not gem: if not gem:
euscan.output.eerror("Can't guess gem name using %s and %s" % \ output.eerror("Can't guess gem name using %s and %s" % \
(cpv, url)) (cpv, url))
return [] return []
url = 'http://rubygems.org/api/v1/versions/%s.json' % gem url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
euscan.output.einfo("Using: " + url) output.einfo("Using: " + url)
try: try:
fp = helpers.urlopen(url) fp = helpers.urlopen(url)
@ -65,7 +67,7 @@ def scan(cpv, url):
if helpers.version_filtered(cp, ver, pv): if helpers.version_filtered(cp, ver, pv):
continue continue
url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
ret.append((url, pv)) ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
return ret return ret

View File

@ -37,6 +37,18 @@ _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
_v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
def get_version_type(version):
types = []
gentoo_types = ("alpha", "beta", "pre", "rc", "p")
for token in re.findall("[\._-]([a-zA-Z]+)", version):
if token in gentoo_types:
types.append(token)
if types:
return types[0]
return "release"
# Stolen from g-pypi # Stolen from g-pypi
def gentoo_mangle_version(up_pv): def gentoo_mangle_version(up_pv):
"""Convert PV to MY_PV if needed """Convert PV to MY_PV if needed
@ -121,7 +133,7 @@ def gentoo_mangle_version(up_pv):
pv = up_pv = rev_match.group(1) pv = up_pv = rev_match.group(1)
replace_me = rev_match.group(2) replace_me = rev_match.group(2)
rev = rev_match.group(3) rev = rev_match.group(3)
additional_version = '.' + rev additional_version = '_p' + rev
for this_suf in suf_matches.keys(): for this_suf in suf_matches.keys():
if rs_match: if rs_match:

View File

@ -1,5 +1,5 @@
import os import os
import sys from datetime import datetime
import portage import portage
from portage.dbapi import porttree from portage.dbapi import porttree
@ -10,8 +10,7 @@ from gentoolkit.package import Package
from gentoolkit.eclean.search import (port_settings) from gentoolkit.eclean.search import (port_settings)
from euscan import CONFIG, BLACKLIST_PACKAGES from euscan import CONFIG, BLACKLIST_PACKAGES
from euscan import handlers from euscan import handlers, helpers, output
from euscan import helpers
from euscan.ebuild import package_from_ebuild from euscan.ebuild import package_from_ebuild
import euscan import euscan
@ -19,19 +18,27 @@ import euscan
def filter_versions(cp, versions): def filter_versions(cp, versions):
filtered = {} filtered = {}
for url, version in versions: for url, version, handler, confidence in versions:
''' Try to keep the most specific urls (determinted by the length) ''' # Try to keep the most specific urls (determinted by the length)
if version in filtered and len(url) < len(filtered[version]): if version in filtered and len(url) < len(filtered[version]):
continue continue
''' Remove blacklisted versions ''' # Remove blacklisted versions
if helpers.version_blacklisted(cp, version): if helpers.version_blacklisted(cp, version):
continue continue
filtered[version] = url filtered[version] = {
"url": url,
"handler": handler,
"confidence": confidence
}
return [(cp, filtered[version], version) for version in filtered] return [
(cp, filtered[version]["url"], version, filtered[version]["handler"],
filtered[version]["confidence"])
for version in filtered
]
def scan_upstream_urls(cpv, urls): def scan_upstream_urls(cpv, urls):
@ -39,22 +46,22 @@ def scan_upstream_urls(cpv, urls):
for filename in urls: for filename in urls:
for url in urls[filename]: for url in urls[filename]:
if not CONFIG['quiet']: if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint() pp.uprint()
euscan.output.einfo("SRC_URI is '%s'" % url) output.einfo("SRC_URI is '%s'" % url)
if '://' not in url: if '://' not in url:
euscan.output.einfo("Invalid url '%s'" % url) output.einfo("Invalid url '%s'" % url)
continue continue
''' Try normal scan ''' # Try normal scan
if CONFIG["scan-dir"]: if CONFIG["scan-dir"]:
versions.extend(handlers.scan(cpv, url)) versions.extend(handlers.scan(cpv, url))
if versions and CONFIG['oneshot']: if versions and CONFIG['oneshot']:
break break
''' Brute Force ''' # Brute Force
if CONFIG["brute-force"] > 0: if CONFIG["brute-force"] > 0:
versions.extend(handlers.brute_force(cpv, url)) versions.extend(handlers.brute_force(cpv, url))
@ -91,10 +98,10 @@ def scan_upstream(query):
) )
if not matches: if not matches:
sys.stderr.write( output.ewarn(
pp.warn("No package matching '%s'" % pp.pkgquery(query)) pp.warn("No package matching '%s'" % pp.pkgquery(query))
) )
return [] return None
matches = sorted(matches) matches = sorted(matches)
pkg = matches.pop() pkg = matches.pop()
@ -103,29 +110,42 @@ def scan_upstream(query):
pkg = matches.pop() pkg = matches.pop()
if not pkg: if not pkg:
sys.stderr.write(pp.warn("Package '%s' only have a dev version (9999)" output.ewarn(
% pp.pkgquery(pkg.cp))) pp.warn("Package '%s' only have a dev version (9999)"
return [] % pp.pkgquery(pkg.cp))
)
return None
# useful data only for formatted output
start_time = datetime.now()
output.metadata("datetime", start_time.isoformat(), show=False)
output.metadata("cp", pkg.cp, show=False)
output.metadata("cpv", pkg.cpv, show=False)
if pkg.cp in BLACKLIST_PACKAGES: if pkg.cp in BLACKLIST_PACKAGES:
sys.stderr.write( output.ewarn(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)) pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
) )
return [] return None
if not CONFIG['quiet']: if not CONFIG['quiet']:
if not CONFIG['format']:
pp.uprint( pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
) )
pp.uprint() pp.uprint()
else:
output.metadata("overlay", pp.section(pkg.repo_name()))
ebuild_path = pkg.ebuild_path() ebuild_path = pkg.ebuild_path()
if ebuild_path: if ebuild_path:
pp.uprint('Ebuild: ' + pp.path(os.path.normpath(ebuild_path))) output.metadata(
"ebuild", pp.path(os.path.normpath(ebuild_path))
)
pp.uprint('Repository: ' + pkg.repo_name()) output.metadata("repository", pkg.repo_name())
pp.uprint('Homepage: ' + pkg.environment("HOMEPAGE")) output.metadata("homepage", pkg.environment("HOMEPAGE"))
pp.uprint('Description: ' + pkg.environment("DESCRIPTION")) output.metadata("description", pkg.environment("DESCRIPTION"))
cpv = pkg.cpv cpv = pkg.cpv
metadata = { metadata = {
@ -137,15 +157,19 @@ def scan_upstream(query):
alist = porttree._parse_uri_map(cpv, metadata, use=use) alist = porttree._parse_uri_map(cpv, metadata, use=use)
aalist = porttree._parse_uri_map(cpv, metadata) aalist = porttree._parse_uri_map(cpv, metadata)
except Exception as e: except Exception as e:
sys.stderr.write(pp.warn("%s\n" % str(e))) output.ewarn(pp.warn("%s\n" % str(e)))
sys.stderr.write( output.ewarn(
pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)) pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
) )
return [] return None
if "mirror" in portage.settings.features: if "mirror" in portage.settings.features:
urls = aalist urls = aalist
else: else:
urls = alist urls = alist
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
return scan_upstream_urls(pkg.cpv, urls) return scan_upstream_urls(pkg.cpv, urls)