diff --git a/bin/euscan b/bin/euscan index e7708e3..dfdedca 100755 --- a/bin/euscan +++ b/bin/euscan @@ -6,51 +6,58 @@ Distributed under the terms of the GNU General Public License v2 from __future__ import print_function -""" Meta """ + +# Meta + __author__ = "Corentin Chary (iksaif)" __email__ = "corentin.chary@gmail.com" __version__ = "git" __productname__ = "euscan" __description__ = "A tool to detect new upstream releases." -__version__ = "git" -""" Imports """ + +# Imports import sys import getopt import errno import httplib -from portage.output import white, yellow, turquoise, green, EOutput +from portage.output import white, yellow, turquoise, green from portage.exception import AmbiguousPackageName from gentoolkit import pprinter as pp from gentoolkit.eclean.search import (port_settings) from gentoolkit.errors import GentoolkitException -import euscan -from euscan import CONFIG +from euscan import CONFIG, output from euscan.scan import scan_upstream -""" Globals """ + +# Globals + +def exit_helper(status): + if CONFIG["format"]: + print(output.get_formatted_output()) + sys.exit(status) -def setupSignals(): - """ This block ensures that ^C interrupts are handled quietly. """ +def setup_signals(): + """This block ensures that ^C interrupts are handled quietly.""" import signal def exithandler(signum, frame): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) print() - sys.exit(errno.EINTR) + exit_helper(errno.EINTR) signal.signal(signal.SIGINT, exithandler) signal.signal(signal.SIGTERM, exithandler) signal.signal(signal.SIGPIPE, signal.SIG_DFL) -def printVersion(): +def print_version(): """Output the version info.""" print("%s (%s) - %s" \ % (__productname__, __version__, __description__)) @@ -60,23 +67,26 @@ def printVersion(): print("Distributed under the terms of the GNU General Public License v2") -def printUsage(_error=None, help=None): +def print_usage(_error=None, help=None): """Print help message. May also print partial help to stderr if an error from {'options'} is specified.""" out = sys.stdout if _error: out = sys.stderr + if not _error in ('global-options', 'packages',): _error = None + if not _error and not help: help = 'all' + if _error in ('global-options',): - print(pp.error("Wrong option on command line."), file=out) - print(file=out) + output.eerror("Wrong option on command line.\n") + if _error in ('packages',): - print(pp.error("You need to specify exactly one package."), file=out) - print(file=out) + output.eerror("You need to specify exactly one package.\n") + print(white("Usage:"), file=out) if _error in ('global-options', 'packages',) or help == 'all': print(" " + turquoise(__productname__), @@ -106,14 +116,19 @@ def printUsage(_error=None, help=None): " (default: 2)\n" + " " * 29 + "bigger levels will generate more versions numbers\n" + " " * 29 + "0 means disabled", file=out) + print(yellow(" -f, --format=") + + " - define the output " + yellow("") + + " (available: json)", file=out) print(file=out) + if _error in ('packages',) or help: print(green(" package") + " - the packages (or ebuilds) you want to scan", file=out) print(file=out) - '''print( "More detailed instruction can be found in", - turquoise("`man %s`" % __productname__), file=out)''' + + #print( "More detailed instruction can be found in", + #turquoise("`man %s`" % __productname__), file=out) class ParseArgsException(Exception): @@ -125,12 +140,12 @@ class ParseArgsException(Exception): return repr(self.value) -def parseArgs(): +def parse_args(): """Parse the command line arguments. Raise exceptions on - errors. Returns package and affect the CONFIG dict. + errors. Returns packages and affects the CONFIG dict. """ - def optionSwitch(opts): + def option_switch(opts): """local function for interpreting command line options and setting options accordingly""" return_code = True @@ -150,30 +165,36 @@ def parseArgs(): elif o in ("-b", "--brute-force"): CONFIG['brute-force'] = int(a) elif o in ("-v", "--verbose") and not CONFIG['quiet']: - CONFIG['verbose'] += 1 + CONFIG['verbose'] += 1 + elif o in ("-f", "--format"): + CONFIG['format'] = a + CONFIG['nocolor'] = True + pp.output.nocolor() else: return_code = False return return_code - ' here are the different allowed command line options (getopt args) ' + # here are the different allowed command line options (getopt args) getopt_options = {'short': {}, 'long': {}} - getopt_options['short']['global'] = "hVCqv1b:" - getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet", - "verbose", "oneshot", "brute-force="] + getopt_options['short']['global'] = "hVCqv1bf:" + getopt_options['long']['global'] = [ + "help", "version", "nocolor", "quiet", "verbose", "oneshot", + "brute-force=", "format=" + ] short_opts = getopt_options['short']['global'] long_opts = getopt_options['long']['global'] opts_mode = 'global' - ' apply getopts to command line, show partial help on failure ' + # apply getopts to command line, show partial help on failure try: opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) except: raise ParseArgsException(opts_mode + '-options') - ' set options accordingly ' - optionSwitch(opts) + # set options accordingly + option_switch(opts) if len(args) < 1: raise ParseArgsException('packages') @@ -183,81 +204,80 @@ def parseArgs(): def main(): """Parse command line and execute all actions.""" - CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true') - or not sys.stdout.isatty()) + CONFIG['nocolor'] = ( + port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty() + ) if CONFIG['nocolor']: pp.output.nocolor() - ' parse command line options and actions ' + + # parse command line options and actions try: - packages = parseArgs() + queries = parse_args() except ParseArgsException as e: if e.value == 'help': - printUsage(help='all') - sys.exit(0) - elif e.value[:5] == 'help-': - printUsage(help=e.value[5:]) - sys.exit(0) - elif e.value == 'version': - printVersion() - sys.exit(0) - else: - printUsage(e.value) - sys.exit(errno.EINVAL) + print_usage(help='all') + exit_helper(0) + + elif e.value[:5] == 'help-': + print_usage(help=e.value[5:]) + exit_helper(0) + + elif e.value == 'version': + print_version() + exit_helper(0) + + else: + print_usage(e.value) + exit_helper(errno.EINVAL) - """ Change euscan's output """ - euscan.output = EOutput(CONFIG['quiet']) if CONFIG['verbose'] > 2: httplib.HTTPConnection.debuglevel = 1 - for package in packages: + for query in queries: ret = [] + output.set_query(query) + try: - ret = scan_upstream(package) + ret = scan_upstream(query) except AmbiguousPackageName as e: pkgs = e.args[0] - for candidate in pkgs: - print(candidate) + output.eerror("\n".join(pkgs)) from os.path import basename # To get the short name - print(file=sys.stderr) - print( - pp.error( - "The short ebuild name '%s' is ambiguous. Please specify" \ - % basename(pkgs[0]) - ), - file=sys.stderr, end="" + output.error( + "The short ebuild name '%s' is ambiguous. Please specify" % + basename(pkgs[0]), + "one of the above fully-qualified ebuild names instead." ) - pp.die(1, "one of the above fully-qualified ebuild names instead.") - except GentoolkitException as err: - pp.die(1, '%s: %s' % (package, str(err))) - except Exception as err: - pp.die(1, '%s: %s' % (package, str(err))) + exit_helper(1) - if not CONFIG['quiet']: + except GentoolkitException as err: + output.eerror('%s: %s' % (query, str(err))) + exit_helper(1) + + except Exception as err: + output.eerror('%s: %s' % (query, str(err))) + exit_helper(1) + + if not CONFIG['quiet'] and not CONFIG['format']: print() - for cp, url, version in ret: - if not CONFIG['quiet']: - print("Upstream Version: " - + pp.number("%s" % version) - + pp.path(" %s" % url)) - else: - print(pp.cpv("%s-%s" % (cp, version)) - + ": " + pp.path(url)) + if ret is not None: + if len(ret) > 0: + for cp, url, version, handler, confidence in ret: + output.result(cp, version, url, handler, confidence) + elif not CONFIG['quiet']: + output.ewarn( + "Didn't find any new version, check package's homepage " + + "for more informations" + ) - if not len(ret) and not CONFIG['quiet']: - print(pp.warn("Didn't find any new version, " - + "check package's homepage for " - + "more informations")) + output.set_query(None) if __name__ == "__main__": - try: - setupSignals() - main() - except KeyboardInterrupt: - print("Aborted.") - sys.exit(errno.EINTR) - sys.exit(0) + setup_signals() + main() + exit_helper(0) diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index 391a7a3..bb83dc8 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -3,11 +3,17 @@ # Copyright 2011 Corentin Chary # Distributed under the terms of the GNU General Public License v2 +from io import StringIO +from collections import defaultdict +import json + +from gentoolkit import pprinter as pp +from portage.output import EOutput + + __version__ = "git" -from portage.output import EOutput - CONFIG = { 'nocolor': False, 'quiet': False, @@ -20,11 +26,11 @@ CONFIG = { 'oneshot': True, 'user-agent': 'escan (http://euscan.iksaif.net)', 'skip-robots-txt': False, - 'cache': False + 'cache': False, + 'format': None, + 'indent': 2 } -output = EOutput(CONFIG['quiet']) - BLACKLIST_VERSIONS = [ # Compatibility package for running binaries linked against a # pre gcc 3.4 libstdc++, won't be updated @@ -53,10 +59,11 @@ BRUTEFORCE_BLACKLIST_PACKAGES = [ BRUTEFORCE_BLACKLIST_URLS = [ 'http://(.*)dockapps.org/download.php/id/(.*)', # infinite loop 'http://hydra.nixos.org/build/(.*)', # infinite loop - 'http://www.rennings.net/gentoo/distfiles/(.*)', # Doesn't respect 404, infinite loop - 'http://art.gnome.org/download/(.*)', # Doesn't respect 404, infinite loop - 'http://barelysufficient.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop - 'http://olemarkus.org/~olemarkus/(.*)', # Doesn't respect 404, infinite loop + # Doesn't respect 404, infinite loop + 'http://www.rennings.net/gentoo/distfiles/(.*)', + 'http://art.gnome.org/download/(.*)', + 'http://barelysufficient.org/~olemarkus/(.*)', + 'http://olemarkus.org/~olemarkus/(.*)', ] ROBOTS_TXT_BLACKLIST_DOMAINS = [ @@ -67,3 +74,94 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [ '(.*)chromium.org(.*)', '(.*)nodejs.org(.*)', ] + + +class EOutputFile(EOutput): + """ + Override of EOutput, allows to specify an output file for writes + """ + def __init__(self, out_file=None, *args, **kwargs): + super(EOutputFile, self).__init__(*args, **kwargs) + self.out_file = out_file + + def _write(self, f, msg): + if self.out_file is None: + super(EOutputFile, self)._write(f, msg) + else: + super(EOutputFile, self)._write(self.out_file, msg) + + +class EuscanOutput(object): + """ + Class that handles output for euscan + """ + def __init__(self, config): + self.config = config + self.queries = defaultdict(dict) + self.current_query = None + + def set_query(self, query): + self.current_query = query + if query is not None: + if not query in self.queries: + self.queries[query] = { + "messages": defaultdict(StringIO), + "result": [], + "metadata": {}, + } + + def get_formatted_output(self): + data = {} + + for query in self.queries: + data[query] = { + "result": self.queries[query]["result"], + "metadata": self.queries[query]["metadata"], + "messages": {} + } + for key in self.queries[query]["messages"]: + if key not in ("ebegin", "eend"): + _msg = self.queries[query]["messages"][key].getvalue() + val = [x for x in _msg.split("\n") if x] + data[query]["messages"][key] = val + + if self.config["format"].lower() == "json": + return json.dumps(data, indent=self.config["indent"]) + else: + raise TypeError("Invalid output format") + + def result(self, cp, version, url, handler, confidence): + from euscan.helpers import get_version_type + + if self.config['format']: + _curr = self.queries[self.current_query] + _curr["result"].append( + {"version": version, "urls": [url], "handler": handler, + "confidence": confidence, "type": get_version_type(version)} + ) + else: + if not self.config['quiet']: + print "Upstream Version:", pp.number("%s" % version), + print pp.path(" %s" % url) + else: + print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url) + + def metadata(self, key, value, show=True): + if self.config["format"]: + self.queries[self.current_query]["metadata"][key] = value + elif show: + print "%s: %s" % (key.capitalize(), value) + + def __getattr__(self, key): + if self.config["format"]: + out_file = self.queries[self.current_query]["messages"][key] + + _output = EOutputFile(out_file=out_file, + quiet=self.config['quiet']) + ret = getattr(_output, key) + else: + ret = getattr(EOutputFile(quiet=self.config['quiet']), key) + return ret + + +output = EuscanOutput(CONFIG) diff --git a/pym/euscan/handlers/cpan.py b/pym/euscan/handlers/cpan.py index 1b5a111..7dcc246 100644 --- a/pym/euscan/handlers/cpan.py +++ b/pym/euscan/handlers/cpan.py @@ -3,8 +3,10 @@ import portage import urllib2 import json -from euscan import helpers -import euscan +from euscan import helpers, output + +HANDLER_NAME = "cpan" +CONFIDENCE = 100.0 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*") @@ -83,7 +85,7 @@ def scan(cpv, url): orig_url = url url = 'http://search.cpan.org/api/dist/%s' % pkg - euscan.output.einfo("Using: " + url) + output.einfo("Using: " + url) try: fp = helpers.urlopen(url) @@ -125,7 +127,7 @@ def scan(cpv, url): if url == orig_url: continue - ret.append((url, pv)) + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/pym/euscan/handlers/generic.py b/pym/euscan/handlers/generic.py index ceb854b..5cf9d49 100644 --- a/pym/euscan/handlers/generic.py +++ b/pym/euscan/handlers/generic.py @@ -7,9 +7,13 @@ from BeautifulSoup import BeautifulSoup import portage from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \ - BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS -from euscan import helpers -import euscan + BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers + +HANDLER_NAME = "generic" +CONFIDENCE = 50.0 + +BRUTEFORCE_HANDLER_NAME = "brute_force" +BRUTEFORCE_CONFIDENCE = 30.0 def scan_html(data, url, pattern): @@ -53,7 +57,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): steps = steps[1:] - euscan.output.einfo("Scanning: %s" % url) + output.einfo("Scanning: %s" % url) try: fp = helpers.urlopen(url) @@ -87,7 +91,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): path = url + path if not steps and path not in orig_url: - versions.append((path, pv)) + versions.append((path, pv, HANDLER_NAME, CONFIDENCE)) if steps: ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url) @@ -99,7 +103,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url): def scan(cpv, url): for bu in SCANDIR_BLACKLIST_URLS: if re.match(bu, url): - euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu)) + output.einfo("%s is blacklisted by rule %s" % (url, bu)) return [] resolved_url = helpers.parse_mirror(url) @@ -112,23 +116,25 @@ def scan(cpv, url): if ver not in resolved_url: newver = helpers.version_change_end_sep(ver) if newver and newver in resolved_url: - euscan.output.einfo( + output.einfo( "Version: using %s instead of %s" % (newver, ver) ) ver = newver template = helpers.template_from_url(resolved_url, ver) if '${' not in template: - euscan.output.einfo( + output.einfo( "Url doesn't seems to depend on version: %s not found in %s" % (ver, resolved_url) ) return [] else: - euscan.output.einfo("Scanning: %s" % template) + output.einfo("Scanning: %s" % template) steps = helpers.generate_scan_paths(template) - return scan_directory_recursive(cp, ver, rev, "", steps, url) + ret = scan_directory_recursive(cp, ver, rev, "", steps, url) + + return ret def brute_force(cpv, url): @@ -140,37 +146,37 @@ def brute_force(cpv, url): for bp in BRUTEFORCE_BLACKLIST_PACKAGES: if re.match(bp, cp): - euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp)) + output.einfo("%s is blacklisted by rule %s" % (cp, bp)) return [] for bp in BRUTEFORCE_BLACKLIST_URLS: if re.match(bp, url): - euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp)) + output.einfo("%s is blacklisted by rule %s" % (cp, bp)) return [] - euscan.output.einfo("Generating version from " + ver) + output.einfo("Generating version from " + ver) components = helpers.split_version(ver) versions = helpers.gen_versions(components, CONFIG["brute-force"]) - """ Remove unwanted versions """ + # Remove unwanted versions for v in versions: if helpers.vercmp(cp, ver, helpers.join_version(v)) >= 0: versions.remove(v) if not versions: - euscan.output.einfo("Can't generate new versions from " + ver) + output.einfo("Can't generate new versions from " + ver) return [] template = helpers.template_from_url(url, ver) if '${PV}' not in template: - euscan.output.einfo( + output.einfo( "Url doesn't seems to depend on full version: %s not found in %s" % (ver, url)) return [] else: - euscan.output.einfo("Brute forcing: %s" % template) + output.einfo("Brute forcing: %s" % template) result = [] @@ -195,10 +201,11 @@ def brute_force(cpv, url): if not infos: continue - result.append([url, version]) + result.append([url, version, BRUTEFORCE_HANDLER_NAME, + BRUTEFORCE_CONFIDENCE]) if len(result) > CONFIG['brute-force-false-watermark']: - euscan.output.einfo( + output.einfo( "Broken server detected ! Skipping brute force." ) return [] diff --git a/pym/euscan/handlers/kde.py b/pym/euscan/handlers/kde.py index 00947c0..2b27639 100644 --- a/pym/euscan/handlers/kde.py +++ b/pym/euscan/handlers/kde.py @@ -1,5 +1,7 @@ from euscan.handlers import generic +HANDLER_NAME = "kde" + def can_handle(cpv, url): if url.startswith('mirror://kde/'): @@ -10,10 +12,10 @@ def can_handle(cpv, url): def clean_results(results): ret = [] - for path, version in results: + for path, version, confidence in results: if version == '5SUMS': continue - ret.append((path, version)) + ret.append((path, version, HANDLER_NAME, confidence)) return ret diff --git a/pym/euscan/handlers/php.py b/pym/euscan/handlers/php.py index a4c7267..82c49c5 100644 --- a/pym/euscan/handlers/php.py +++ b/pym/euscan/handlers/php.py @@ -3,8 +3,10 @@ import portage import urllib2 import xml.dom.minidom -from euscan import helpers -import euscan +from euscan import helpers, output + +HANDLER_NAME = "php" +CONFIDENCE = 100.0 def can_handle(cpv, url): @@ -34,7 +36,7 @@ def scan(cpv, url): orig_url = url url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower()) - euscan.output.einfo("Using: " + url) + output.einfo("Using: " + url) try: fp = helpers.urlopen(url) @@ -64,7 +66,7 @@ def scan(cpv, url): if url == orig_url: continue - ret.append((url, pv)) + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/pym/euscan/handlers/pypi.py b/pym/euscan/handlers/pypi.py index 1e477b7..0b3ed73 100644 --- a/pym/euscan/handlers/pypi.py +++ b/pym/euscan/handlers/pypi.py @@ -3,8 +3,10 @@ import re import portage -from euscan import helpers -import euscan +from euscan import helpers, output + +HANDLER_NAME = "pypi" +CONFIDENCE = 100.0 def can_handle(cpv, url): @@ -26,7 +28,7 @@ def scan(cpv, url): package = guess_package(cpv, url) - euscan.output.einfo("Using PyPi XMLRPC: " + package) + output.einfo("Using PyPi XMLRPC: " + package) client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') versions = client.package_releases(package) @@ -46,7 +48,7 @@ def scan(cpv, url): continue urls = client.release_urls(package, up_pv) urls = " ".join([infos['url'] for infos in urls]) - ret.append((urls, pv)) + ret.append((urls, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/pym/euscan/handlers/rubygem.py b/pym/euscan/handlers/rubygem.py index ec57a2a..8bbd5ab 100644 --- a/pym/euscan/handlers/rubygem.py +++ b/pym/euscan/handlers/rubygem.py @@ -3,8 +3,10 @@ import portage import json import urllib2 -from euscan import helpers -import euscan +from euscan import helpers, output + +HANDLER_NAME = "rubygem" +CONFIDENCE = 100.0 def can_handle(cpv, url): @@ -31,13 +33,13 @@ def scan(cpv, url): gem = guess_gem(cpv, url) if not gem: - euscan.output.eerror("Can't guess gem name using %s and %s" % \ + output.eerror("Can't guess gem name using %s and %s" % \ (cpv, url)) return [] url = 'http://rubygems.org/api/v1/versions/%s.json' % gem - euscan.output.einfo("Using: " + url) + output.einfo("Using: " + url) try: fp = helpers.urlopen(url) @@ -65,7 +67,7 @@ def scan(cpv, url): if helpers.version_filtered(cp, ver, pv): continue url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv) - ret.append((url, pv)) + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/pym/euscan/helpers.py b/pym/euscan/helpers.py index ab0fcda..6a96152 100644 --- a/pym/euscan/helpers.py +++ b/pym/euscan/helpers.py @@ -37,6 +37,18 @@ _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)' _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))' +def get_version_type(version): + types = [] + gentoo_types = ("alpha", "beta", "pre", "rc", "p") + + for token in re.findall("[\._-]([a-zA-Z]+)", version): + if token in gentoo_types: + types.append(token) + if types: + return types[0] + return "release" + + # Stolen from g-pypi def gentoo_mangle_version(up_pv): """Convert PV to MY_PV if needed @@ -121,7 +133,7 @@ def gentoo_mangle_version(up_pv): pv = up_pv = rev_match.group(1) replace_me = rev_match.group(2) rev = rev_match.group(3) - additional_version = '.' + rev + additional_version = '_p' + rev for this_suf in suf_matches.keys(): if rs_match: diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 10628b5..a73bf76 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -1,5 +1,5 @@ import os -import sys +from datetime import datetime import portage from portage.dbapi import porttree @@ -10,8 +10,7 @@ from gentoolkit.package import Package from gentoolkit.eclean.search import (port_settings) from euscan import CONFIG, BLACKLIST_PACKAGES -from euscan import handlers -from euscan import helpers +from euscan import handlers, helpers, output from euscan.ebuild import package_from_ebuild import euscan @@ -19,19 +18,27 @@ import euscan def filter_versions(cp, versions): filtered = {} - for url, version in versions: + for url, version, handler, confidence in versions: - ''' Try to keep the most specific urls (determinted by the length) ''' + # Try to keep the most specific urls (determinted by the length) if version in filtered and len(url) < len(filtered[version]): continue - ''' Remove blacklisted versions ''' + # Remove blacklisted versions if helpers.version_blacklisted(cp, version): continue - filtered[version] = url + filtered[version] = { + "url": url, + "handler": handler, + "confidence": confidence + } - return [(cp, filtered[version], version) for version in filtered] + return [ + (cp, filtered[version]["url"], version, filtered[version]["handler"], + filtered[version]["confidence"]) + for version in filtered + ] def scan_upstream_urls(cpv, urls): @@ -39,22 +46,22 @@ def scan_upstream_urls(cpv, urls): for filename in urls: for url in urls[filename]: - if not CONFIG['quiet']: + if not CONFIG['quiet'] and not CONFIG['format']: pp.uprint() - euscan.output.einfo("SRC_URI is '%s'" % url) + output.einfo("SRC_URI is '%s'" % url) if '://' not in url: - euscan.output.einfo("Invalid url '%s'" % url) + output.einfo("Invalid url '%s'" % url) continue - ''' Try normal scan ''' + # Try normal scan if CONFIG["scan-dir"]: versions.extend(handlers.scan(cpv, url)) if versions and CONFIG['oneshot']: break - ''' Brute Force ''' + # Brute Force if CONFIG["brute-force"] > 0: versions.extend(handlers.brute_force(cpv, url)) @@ -91,10 +98,10 @@ def scan_upstream(query): ) if not matches: - sys.stderr.write( + output.ewarn( pp.warn("No package matching '%s'" % pp.pkgquery(query)) ) - return [] + return None matches = sorted(matches) pkg = matches.pop() @@ -103,29 +110,42 @@ def scan_upstream(query): pkg = matches.pop() if not pkg: - sys.stderr.write(pp.warn("Package '%s' only have a dev version (9999)" - % pp.pkgquery(pkg.cp))) - return [] + output.ewarn( + pp.warn("Package '%s' only have a dev version (9999)" + % pp.pkgquery(pkg.cp)) + ) + return None + + # useful data only for formatted output + start_time = datetime.now() + output.metadata("datetime", start_time.isoformat(), show=False) + output.metadata("cp", pkg.cp, show=False) + output.metadata("cpv", pkg.cpv, show=False) if pkg.cp in BLACKLIST_PACKAGES: - sys.stderr.write( + output.ewarn( pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)) ) - return [] + return None if not CONFIG['quiet']: - pp.uprint( - " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) - ) - pp.uprint() + if not CONFIG['format']: + pp.uprint( + " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) + ) + pp.uprint() + else: + output.metadata("overlay", pp.section(pkg.repo_name())) ebuild_path = pkg.ebuild_path() if ebuild_path: - pp.uprint('Ebuild: ' + pp.path(os.path.normpath(ebuild_path))) + output.metadata( + "ebuild", pp.path(os.path.normpath(ebuild_path)) + ) - pp.uprint('Repository: ' + pkg.repo_name()) - pp.uprint('Homepage: ' + pkg.environment("HOMEPAGE")) - pp.uprint('Description: ' + pkg.environment("DESCRIPTION")) + output.metadata("repository", pkg.repo_name()) + output.metadata("homepage", pkg.environment("HOMEPAGE")) + output.metadata("description", pkg.environment("DESCRIPTION")) cpv = pkg.cpv metadata = { @@ -137,15 +157,19 @@ def scan_upstream(query): alist = porttree._parse_uri_map(cpv, metadata, use=use) aalist = porttree._parse_uri_map(cpv, metadata) except Exception as e: - sys.stderr.write(pp.warn("%s\n" % str(e))) - sys.stderr.write( + output.ewarn(pp.warn("%s\n" % str(e))) + output.ewarn( pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv)) ) - return [] + return None if "mirror" in portage.settings.features: urls = aalist else: urls = alist + # output scan time for formatted output + scan_time = (datetime.now() - start_time).total_seconds() + output.metadata("scan_time", scan_time, show=False) + return scan_upstream_urls(pkg.cpv, urls)