Merge git://github.com/volpino/euscan

Conflicts:
	pym/euscan/scan.py

bin/euscan

@@ -6,51 +6,58 @@ Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
 
-""" Meta """
+
+# Meta
+
 __author__ = "Corentin Chary (iksaif)"
 __email__ = "corentin.chary@gmail.com"
 __version__ = "git"
 __productname__ = "euscan"
 __description__ = "A tool to detect new upstream releases."
-__version__ = "git"
 
-""" Imports """
+
+# Imports
 
 import sys
 import getopt
 import errno
 import httplib
 
-from portage.output import white, yellow, turquoise, green, EOutput
+from portage.output import white, yellow, turquoise, green
 from portage.exception import AmbiguousPackageName
 
 from gentoolkit import pprinter as pp
 from gentoolkit.eclean.search import (port_settings)
 from gentoolkit.errors import GentoolkitException
 
-import euscan
+from euscan import CONFIG, output
-from euscan import CONFIG
 from euscan.scan import scan_upstream
 
-""" Globals """
+
+# Globals
+
+def exit_helper(status):
+    if CONFIG["format"]:
+        print(output.get_formatted_output())
+    sys.exit(status)
 
 
-def setupSignals():
+def setup_signals():
-    """ This block ensures that ^C interrupts are handled quietly. """
+    """This block ensures that ^C interrupts are handled quietly."""
     import signal
 
     def exithandler(signum, frame):
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         signal.signal(signal.SIGTERM, signal.SIG_IGN)
         print()
-        sys.exit(errno.EINTR)
+        exit_helper(errno.EINTR)
 
     signal.signal(signal.SIGINT, exithandler)
     signal.signal(signal.SIGTERM, exithandler)
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
 
-def printVersion():
+def print_version():
     """Output the version info."""
     print("%s (%s) - %s" \
             % (__productname__, __version__, __description__))
@@ -60,23 +67,26 @@ def printVersion():
     print("Distributed under the terms of the GNU General Public License v2")
 
 
-def printUsage(_error=None, help=None):
+def print_usage(_error=None, help=None):
     """Print help message. May also print partial help to stderr if an
     error from {'options'} is specified."""
 
     out = sys.stdout
     if _error:
         out = sys.stderr
+
     if not _error in ('global-options', 'packages',):
         _error = None
+
     if not _error and not help:
         help = 'all'
+
     if _error in ('global-options',):
-        print(pp.error("Wrong option on command line."), file=out)
+        output.eerror("Wrong option on command line.\n")
-        print(file=out)
+
     if _error in ('packages',):
-        print(pp.error("You need to specify exactly one package."), file=out)
+        output.eerror("You need to specify exactly one package.\n")
-        print(file=out)
+
     print(white("Usage:"), file=out)
     if _error in ('global-options', 'packages',) or help == 'all':
         print(" " + turquoise(__productname__),
@@ -106,14 +116,19 @@ def printUsage(_error=None, help=None):
             " (default: 2)\n" +
             " " * 29 + "bigger levels will generate more versions numbers\n" +
             " " * 29 + "0 means disabled", file=out)
+        print(yellow(" -f, --format=<format>") +
+              "     - define the output " + yellow("<format>") +
+              " (available: json)", file=out)
         print(file=out)
+
     if _error in ('packages',) or help:
         print(green(" package") +
             "                   - the packages (or ebuilds) you want to scan",
             file=out)
         print(file=out)
-        '''print( "More detailed instruction can be found in",
+
-        turquoise("`man %s`" % __productname__), file=out)'''
+        #print( "More detailed instruction can be found in",
+        #turquoise("`man %s`" % __productname__), file=out)
 
 
 class ParseArgsException(Exception):
@@ -125,12 +140,12 @@ class ParseArgsException(Exception):
         return repr(self.value)
 
 
-def parseArgs():
+def parse_args():
     """Parse the command line arguments. Raise exceptions on
-    errors. Returns package and affect the CONFIG dict.
+    errors. Returns packages and affects the CONFIG dict.
     """
 
-    def optionSwitch(opts):
+    def option_switch(opts):
         """local function for interpreting command line options
         and setting options accordingly"""
         return_code = True
@@ -151,29 +166,35 @@ def parseArgs():
                 CONFIG['brute-force'] = int(a)
             elif o in ("-v", "--verbose") and not CONFIG['quiet']:
                 CONFIG['verbose'] += 1
+            elif o in ("-f", "--format"):
+                CONFIG['format'] = a
+                CONFIG['nocolor'] = True
+                pp.output.nocolor()
             else:
                 return_code = False
 
         return return_code
 
-    ' here are the different allowed command line options (getopt args) '
+    # here are the different allowed command line options (getopt args)
     getopt_options = {'short': {}, 'long': {}}
-    getopt_options['short']['global'] = "hVCqv1b:"
+    getopt_options['short']['global'] = "hVCqv1bf:"
-    getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet",
+    getopt_options['long']['global'] = [
-                        "verbose", "oneshot", "brute-force="]
+        "help", "version", "nocolor", "quiet", "verbose", "oneshot",
+        "brute-force=", "format="
+    ]
 
     short_opts = getopt_options['short']['global']
     long_opts = getopt_options['long']['global']
     opts_mode = 'global'
 
-    ' apply getopts to command line, show partial help on failure '
+    # apply getopts to command line, show partial help on failure
     try:
         opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
     except:
         raise ParseArgsException(opts_mode + '-options')
 
-    ' set options accordingly '
+    # set options accordingly
-    optionSwitch(opts)
+    option_switch(opts)
 
     if len(args) < 1:
         raise ParseArgsException('packages')
@@ -183,81 +204,80 @@ def parseArgs():
 
 def main():
     """Parse command line and execute all actions."""
-    CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true')
+    CONFIG['nocolor'] = (
-        or not sys.stdout.isatty())
+        port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()
+    )
     if CONFIG['nocolor']:
         pp.output.nocolor()
-    ' parse command line options and actions '
+
+    # parse command line options and actions
     try:
-        packages = parseArgs()
+        queries = parse_args()
     except ParseArgsException as e:
         if e.value == 'help':
-            printUsage(help='all')
+            print_usage(help='all')
-            sys.exit(0)
+            exit_helper(0)
-        elif e.value[:5] == 'help-':
+
-            printUsage(help=e.value[5:])
+        elif e.value[:5] == 'help-':
-            sys.exit(0)
+            print_usage(help=e.value[5:])
-        elif e.value == 'version':
+            exit_helper(0)
-            printVersion()
+
-            sys.exit(0)
+        elif e.value == 'version':
-        else:
+            print_version()
-            printUsage(e.value)
+            exit_helper(0)
-            sys.exit(errno.EINVAL)
+
+        else:
+            print_usage(e.value)
+            exit_helper(errno.EINVAL)
 
-    """ Change euscan's output """
-    euscan.output = EOutput(CONFIG['quiet'])
     if CONFIG['verbose'] > 2:
         httplib.HTTPConnection.debuglevel = 1
 
-    for package in packages:
+    for query in queries:
         ret = []
 
+        output.set_query(query)
+
         try:
-            ret = scan_upstream(package)
+            ret = scan_upstream(query)
         except AmbiguousPackageName as e:
             pkgs = e.args[0]
-            for candidate in pkgs:
+            output.eerror("\n".join(pkgs))
-                print(candidate)
 
             from os.path import basename  # To get the short name
 
-            print(file=sys.stderr)
+            output.error(
-            print(
+                "The short ebuild name '%s' is ambiguous. Please specify" %
-                pp.error(
+                    basename(pkgs[0]),
-                    "The short ebuild name '%s' is ambiguous. Please specify" \
+                "one of the above fully-qualified ebuild names instead."
-                    % basename(pkgs[0])
-                ),
-                file=sys.stderr, end=""
             )
-            pp.die(1, "one of the above fully-qualified ebuild names instead.")
+            exit_helper(1)
-        except GentoolkitException as err:
-            pp.die(1, '%s: %s' % (package, str(err)))
-        except Exception as err:
-            pp.die(1, '%s: %s' % (package, str(err)))
 
-        if not CONFIG['quiet']:
+        except GentoolkitException as err:
+            output.eerror('%s: %s' % (query, str(err)))
+            exit_helper(1)
+
+        except Exception as err:
+            output.eerror('%s: %s' % (query, str(err)))
+            exit_helper(1)
+
+        if not CONFIG['quiet'] and not CONFIG['format']:
             print()
 
-        for cp, url, version in ret:
+        if ret is not None:
-            if not CONFIG['quiet']:
+            if len(ret) > 0:
-                print("Upstream Version: "
+                for cp, url, version, handler, confidence in ret:
-                      + pp.number("%s" % version)
+                    output.result(cp, version, url, handler, confidence)
-                      + pp.path(" %s" % url))
+            elif not CONFIG['quiet']:
-            else:
+                output.ewarn(
-                print(pp.cpv("%s-%s" % (cp, version))
+                    "Didn't find any new version, check package's homepage " +
-                      + ": " + pp.path(url))
+                    "for more informations"
+                )
 
-        if not len(ret) and not CONFIG['quiet']:
+    output.set_query(None)
-            print(pp.warn("Didn't find any new version, "
-                          + "check package's homepage for "
-                          + "more informations"))
 
 
 if __name__ == "__main__":
-    try:
+    setup_signals()
-        setupSignals()
     main()
-    except KeyboardInterrupt:
+    exit_helper(0)
-        print("Aborted.")
-        sys.exit(errno.EINTR)
-    sys.exit(0)

pym/euscan/__init__.py

@@ -3,11 +3,17 @@
 # Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
 # Distributed under the terms of the GNU General Public License v2
 
+from io import StringIO
+from collections import defaultdict
+import json
+
+from gentoolkit import pprinter as pp
+from portage.output import EOutput
+
+
 __version__ = "git"
 
 
-from portage.output import EOutput
-
 CONFIG = {
     'nocolor': False,
     'quiet': False,
@@ -20,11 +26,11 @@ CONFIG = {
     'oneshot': True,
     'user-agent': 'escan (http://euscan.iksaif.net)',
     'skip-robots-txt': False,
-    'cache': False
+    'cache': False,
+    'format': None,
+    'indent': 2
 }
 
-output = EOutput(CONFIG['quiet'])
-
 BLACKLIST_VERSIONS = [
     # Compatibility package for running binaries linked against a
     # pre gcc 3.4 libstdc++, won't be updated
@@ -53,10 +59,11 @@ BRUTEFORCE_BLACKLIST_PACKAGES = [
 BRUTEFORCE_BLACKLIST_URLS = [
     'http://(.*)dockapps.org/download.php/id/(.*)',  # infinite loop
     'http://hydra.nixos.org/build/(.*)',  # infinite loop
-    'http://www.rennings.net/gentoo/distfiles/(.*)',  # Doesn't respect 404, infinite loop
+    # Doesn't respect 404, infinite loop
-    'http://art.gnome.org/download/(.*)',  # Doesn't respect 404, infinite loop
+    'http://www.rennings.net/gentoo/distfiles/(.*)',
-    'http://barelysufficient.org/~olemarkus/(.*)',  # Doesn't respect 404, infinite loop
+    'http://art.gnome.org/download/(.*)',
-    'http://olemarkus.org/~olemarkus/(.*)',  # Doesn't respect 404, infinite loop
+    'http://barelysufficient.org/~olemarkus/(.*)',
+    'http://olemarkus.org/~olemarkus/(.*)',
 ]
 
 ROBOTS_TXT_BLACKLIST_DOMAINS = [
@@ -67,3 +74,94 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
     '(.*)chromium.org(.*)',
     '(.*)nodejs.org(.*)',
 ]
+
+
+class EOutputFile(EOutput):
+    """
+    Override of EOutput, allows to specify an output file for writes
+    """
+    def __init__(self, out_file=None, *args, **kwargs):
+        super(EOutputFile, self).__init__(*args, **kwargs)
+        self.out_file = out_file
+
+    def _write(self, f, msg):
+        if self.out_file is None:
+            super(EOutputFile, self)._write(f, msg)
+        else:
+            super(EOutputFile, self)._write(self.out_file, msg)
+
+
+class EuscanOutput(object):
+    """
+    Class that handles output for euscan
+    """
+    def __init__(self, config):
+        self.config = config
+        self.queries = defaultdict(dict)
+        self.current_query = None
+
+    def set_query(self, query):
+        self.current_query = query
+        if query is not None:
+            if not query in self.queries:
+                self.queries[query] = {
+                    "messages": defaultdict(StringIO),
+                    "result": [],
+                    "metadata": {},
+                }
+
+    def get_formatted_output(self):
+        data = {}
+
+        for query in self.queries:
+            data[query] = {
+                "result": self.queries[query]["result"],
+                "metadata": self.queries[query]["metadata"],
+                "messages": {}
+            }
+            for key in self.queries[query]["messages"]:
+                if key not in ("ebegin", "eend"):
+                    _msg = self.queries[query]["messages"][key].getvalue()
+                    val = [x for x in _msg.split("\n") if x]
+                    data[query]["messages"][key] = val
+
+        if self.config["format"].lower() == "json":
+            return json.dumps(data, indent=self.config["indent"])
+        else:
+            raise TypeError("Invalid output format")
+
+    def result(self, cp, version, url, handler, confidence):
+        from euscan.helpers import get_version_type
+
+        if self.config['format']:
+            _curr = self.queries[self.current_query]
+            _curr["result"].append(
+                {"version": version, "urls": [url], "handler": handler,
+                 "confidence": confidence, "type": get_version_type(version)}
+            )
+        else:
+            if not self.config['quiet']:
+                print "Upstream Version:", pp.number("%s" % version),
+                print pp.path(" %s" % url)
+            else:
+                print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url)
+
+    def metadata(self, key, value, show=True):
+        if self.config["format"]:
+            self.queries[self.current_query]["metadata"][key] = value
+        elif show:
+            print "%s: %s" % (key.capitalize(), value)
+
+    def __getattr__(self, key):
+        if self.config["format"]:
+            out_file = self.queries[self.current_query]["messages"][key]
+
+            _output = EOutputFile(out_file=out_file,
+                                  quiet=self.config['quiet'])
+            ret = getattr(_output, key)
+        else:
+            ret = getattr(EOutputFile(quiet=self.config['quiet']), key)
+        return ret
+
+
+output = EuscanOutput(CONFIG)

pym/euscan/handlers/cpan.py

@@ -3,8 +3,10 @@ import portage
 import urllib2
 import json
 
-from euscan import helpers
+from euscan import helpers, output
-import euscan
+
+HANDLER_NAME = "cpan"
+CONFIDENCE = 100.0
 
 _cpan_package_name_re = re.compile("mirror://cpan/authors/.*/([^/.]*).*")
 
@@ -83,7 +85,7 @@ def scan(cpv, url):
     orig_url = url
     url = 'http://search.cpan.org/api/dist/%s' % pkg
 
-    euscan.output.einfo("Using: " + url)
+    output.einfo("Using: " + url)
 
     try:
         fp = helpers.urlopen(url)
@@ -125,7 +127,7 @@ def scan(cpv, url):
         if url == orig_url:
             continue
 
-        ret.append((url, pv))
+        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
 

pym/euscan/handlers/generic.py

@@ -7,9 +7,13 @@ from BeautifulSoup import BeautifulSoup
 import portage
 
 from euscan import CONFIG, SCANDIR_BLACKLIST_URLS, \
-    BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS
+    BRUTEFORCE_BLACKLIST_PACKAGES, BRUTEFORCE_BLACKLIST_URLS, output, helpers
-from euscan import helpers
+
-import euscan
+HANDLER_NAME = "generic"
+CONFIDENCE = 50.0
+
+BRUTEFORCE_HANDLER_NAME = "brute_force"
+BRUTEFORCE_CONFIDENCE = 30.0
 
 
 def scan_html(data, url, pattern):
@@ -53,7 +57,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
 
     steps = steps[1:]
 
-    euscan.output.einfo("Scanning: %s" % url)
+    output.einfo("Scanning: %s" % url)
 
     try:
         fp = helpers.urlopen(url)
@@ -87,7 +91,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
             path = url + path
 
         if not steps and path not in orig_url:
-            versions.append((path, pv))
+            versions.append((path, pv, HANDLER_NAME, CONFIDENCE))
 
         if steps:
             ret = scan_directory_recursive(cp, ver, rev, path, steps, orig_url)
@@ -99,7 +103,7 @@ def scan_directory_recursive(cp, ver, rev, url, steps, orig_url):
 def scan(cpv, url):
     for bu in SCANDIR_BLACKLIST_URLS:
         if re.match(bu, url):
-            euscan.output.einfo("%s is blacklisted by rule %s" % (url, bu))
+            output.einfo("%s is blacklisted by rule %s" % (url, bu))
             return []
 
     resolved_url = helpers.parse_mirror(url)
@@ -112,23 +116,25 @@ def scan(cpv, url):
     if ver not in resolved_url:
         newver = helpers.version_change_end_sep(ver)
         if newver and newver in resolved_url:
-            euscan.output.einfo(
+            output.einfo(
                 "Version: using %s instead of %s" % (newver, ver)
             )
             ver = newver
 
     template = helpers.template_from_url(resolved_url, ver)
     if '${' not in template:
-        euscan.output.einfo(
+        output.einfo(
             "Url doesn't seems to depend on version: %s not found in %s" %
             (ver, resolved_url)
         )
         return []
     else:
-        euscan.output.einfo("Scanning: %s" % template)
+        output.einfo("Scanning: %s" % template)
 
     steps = helpers.generate_scan_paths(template)
-    return scan_directory_recursive(cp, ver, rev, "", steps, url)
+    ret = scan_directory_recursive(cp, ver, rev, "", steps, url)
+
+    return ret
 
 
 def brute_force(cpv, url):
@@ -140,37 +146,37 @@ def brute_force(cpv, url):
 
     for bp in BRUTEFORCE_BLACKLIST_PACKAGES:
         if re.match(bp, cp):
-            euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp))
+            output.einfo("%s is blacklisted by rule %s" % (cp, bp))
             return []
 
     for bp in BRUTEFORCE_BLACKLIST_URLS:
         if re.match(bp, url):
-            euscan.output.einfo("%s is blacklisted by rule %s" % (cp, bp))
+            output.einfo("%s is blacklisted by rule %s" % (cp, bp))
             return []
 
-    euscan.output.einfo("Generating version from " + ver)
+    output.einfo("Generating version from " + ver)
 
     components = helpers.split_version(ver)
     versions = helpers.gen_versions(components, CONFIG["brute-force"])
 
-    """ Remove unwanted versions """
+    # Remove unwanted versions
     for v in versions:
         if helpers.vercmp(cp, ver, helpers.join_version(v)) >= 0:
             versions.remove(v)
 
     if not versions:
-        euscan.output.einfo("Can't generate new versions from " + ver)
+        output.einfo("Can't generate new versions from " + ver)
         return []
 
     template = helpers.template_from_url(url, ver)
 
     if '${PV}' not in template:
-        euscan.output.einfo(
+        output.einfo(
             "Url doesn't seems to depend on full version: %s not found in %s" %
             (ver, url))
         return []
     else:
-        euscan.output.einfo("Brute forcing: %s" % template)
+        output.einfo("Brute forcing: %s" % template)
 
     result = []
 
@@ -195,10 +201,11 @@ def brute_force(cpv, url):
         if not infos:
             continue
 
-        result.append([url, version])
+        result.append([url, version, BRUTEFORCE_HANDLER_NAME,
+                       BRUTEFORCE_CONFIDENCE])
 
         if len(result) > CONFIG['brute-force-false-watermark']:
-            euscan.output.einfo(
+            output.einfo(
                 "Broken server detected ! Skipping brute force."
             )
             return []

pym/euscan/handlers/kde.py

@@ -1,5 +1,7 @@
 from euscan.handlers import generic
 
+HANDLER_NAME = "kde"
+
 
 def can_handle(cpv, url):
     if url.startswith('mirror://kde/'):
@@ -10,10 +12,10 @@ def can_handle(cpv, url):
 def clean_results(results):
     ret = []
 
-    for path, version in results:
+    for path, version, confidence in results:
         if version == '5SUMS':
             continue
-        ret.append((path, version))
+        ret.append((path, version, HANDLER_NAME, confidence))
 
     return ret
 

pym/euscan/handlers/php.py

@@ -3,8 +3,10 @@ import portage
 import urllib2
 import xml.dom.minidom
 
-from euscan import helpers
+from euscan import helpers, output
-import euscan
+
+HANDLER_NAME = "php"
+CONFIDENCE = 100.0
 
 
 def can_handle(cpv, url):
@@ -34,7 +36,7 @@ def scan(cpv, url):
     orig_url = url
     url = 'http://%s/rest/r/%s/allreleases.xml' % (channel, pkg.lower())
 
-    euscan.output.einfo("Using: " + url)
+    output.einfo("Using: " + url)
 
     try:
         fp = helpers.urlopen(url)
@@ -64,7 +66,7 @@ def scan(cpv, url):
         if url == orig_url:
             continue
 
-        ret.append((url, pv))
+        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
 

pym/euscan/handlers/pypi.py

@@ -3,8 +3,10 @@ import re
 
 import portage
 
-from euscan import helpers
+from euscan import helpers, output
-import euscan
+
+HANDLER_NAME = "pypi"
+CONFIDENCE = 100.0
 
 
 def can_handle(cpv, url):
@@ -26,7 +28,7 @@ def scan(cpv, url):
 
     package = guess_package(cpv, url)
 
-    euscan.output.einfo("Using PyPi XMLRPC: " + package)
+    output.einfo("Using PyPi XMLRPC: " + package)
 
     client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
     versions = client.package_releases(package)
@@ -46,7 +48,7 @@ def scan(cpv, url):
             continue
         urls = client.release_urls(package, up_pv)
         urls = " ".join([infos['url'] for infos in urls])
-        ret.append((urls, pv))
+        ret.append((urls, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
 

pym/euscan/handlers/rubygem.py

@@ -3,8 +3,10 @@ import portage
 import json
 import urllib2
 
-from euscan import helpers
+from euscan import helpers, output
-import euscan
+
+HANDLER_NAME = "rubygem"
+CONFIDENCE = 100.0
 
 
 def can_handle(cpv, url):
@@ -31,13 +33,13 @@ def scan(cpv, url):
 
     gem = guess_gem(cpv, url)
     if not gem:
-        euscan.output.eerror("Can't guess gem name using %s and %s" % \
+        output.eerror("Can't guess gem name using %s and %s" % \
             (cpv, url))
         return []
 
     url = 'http://rubygems.org/api/v1/versions/%s.json' % gem
 
-    euscan.output.einfo("Using: " + url)
+    output.einfo("Using: " + url)
 
     try:
         fp = helpers.urlopen(url)
@@ -65,7 +67,7 @@ def scan(cpv, url):
         if helpers.version_filtered(cp, ver, pv):
             continue
         url = 'http://rubygems.org/gems/%s-%s.gem' % (gem, up_pv)
-        ret.append((url, pv))
+        ret.append((url, pv, HANDLER_NAME, CONFIDENCE))
 
     return ret
 

pym/euscan/helpers.py

@@ -37,6 +37,18 @@ _v_end = '((-|_)(pre|p|beta|b|alpha|a|rc|r)\d*)'
 _v = r'((\d+)((\.\d+)*)([a-zA-Z]*?)(' + _v_end + '*))'
 
 
+def get_version_type(version):
+    types = []
+    gentoo_types = ("alpha", "beta", "pre", "rc", "p")
+
+    for token in re.findall("[\._-]([a-zA-Z]+)", version):
+        if token in gentoo_types:
+            types.append(token)
+    if types:
+        return types[0]
+    return "release"
+
+
 # Stolen from g-pypi
 def gentoo_mangle_version(up_pv):
     """Convert PV to MY_PV if needed
@@ -121,7 +133,7 @@ def gentoo_mangle_version(up_pv):
         pv = up_pv = rev_match.group(1)
         replace_me = rev_match.group(2)
         rev = rev_match.group(3)
-        additional_version = '.' + rev
+        additional_version = '_p' + rev
 
     for this_suf in suf_matches.keys():
         if rs_match:

pym/euscan/scan.py

@@ -1,5 +1,5 @@
 import os
-import sys
+from datetime import datetime
 
 import portage
 from portage.dbapi import porttree
@@ -10,8 +10,7 @@ from gentoolkit.package import Package
 from gentoolkit.eclean.search import (port_settings)
 
 from euscan import CONFIG, BLACKLIST_PACKAGES
-from euscan import handlers
+from euscan import handlers, helpers, output
-from euscan import helpers
 from euscan.ebuild import package_from_ebuild
 
 import euscan
@@ -19,19 +18,27 @@ import euscan
 def filter_versions(cp, versions):
     filtered = {}
 
-    for url, version in versions:
+    for url, version, handler, confidence in versions:
 
-        ''' Try to keep the most specific urls (determinted by the length) '''
+        # Try to keep the most specific urls (determinted by the length)
         if version in filtered and len(url) < len(filtered[version]):
             continue
 
-        ''' Remove blacklisted versions '''
+        # Remove blacklisted versions
         if helpers.version_blacklisted(cp, version):
             continue
 
-        filtered[version] = url
+        filtered[version] = {
+            "url": url,
+            "handler": handler,
+            "confidence": confidence
+        }
 
-    return [(cp, filtered[version], version) for version in filtered]
+    return [
+        (cp, filtered[version]["url"], version, filtered[version]["handler"],
+         filtered[version]["confidence"])
+        for version in filtered
+    ]
 
 
 def scan_upstream_urls(cpv, urls):
@@ -39,22 +46,22 @@ def scan_upstream_urls(cpv, urls):
 
     for filename in urls:
         for url in urls[filename]:
-            if not CONFIG['quiet']:
+            if not CONFIG['quiet'] and not CONFIG['format']:
                 pp.uprint()
-            euscan.output.einfo("SRC_URI is '%s'" % url)
+            output.einfo("SRC_URI is '%s'" % url)
 
             if '://' not in url:
-                euscan.output.einfo("Invalid url '%s'" % url)
+                output.einfo("Invalid url '%s'" % url)
                 continue
 
-            ''' Try normal scan '''
+            # Try normal scan
             if CONFIG["scan-dir"]:
                 versions.extend(handlers.scan(cpv, url))
 
             if versions and CONFIG['oneshot']:
                 break
 
-            ''' Brute Force '''
+            # Brute Force
             if CONFIG["brute-force"] > 0:
                 versions.extend(handlers.brute_force(cpv, url))
 
@@ -91,10 +98,10 @@ def scan_upstream(query):
         )
 
     if not matches:
-        sys.stderr.write(
+        output.ewarn(
             pp.warn("No package matching '%s'" % pp.pkgquery(query))
         )
-        return []
+        return None
 
     matches = sorted(matches)
     pkg = matches.pop()
@@ -103,29 +110,42 @@ def scan_upstream(query):
         pkg = matches.pop()
 
     if not pkg:
-        sys.stderr.write(pp.warn("Package '%s' only have a dev version (9999)"
+        output.ewarn(
-                                 % pp.pkgquery(pkg.cp)))
+            pp.warn("Package '%s' only have a dev version (9999)"
-        return []
+                    % pp.pkgquery(pkg.cp))
+        )
+        return None
+
+    # useful data only for formatted output
+    start_time = datetime.now()
+    output.metadata("datetime", start_time.isoformat(), show=False)
+    output.metadata("cp", pkg.cp, show=False)
+    output.metadata("cpv", pkg.cpv, show=False)
 
     if pkg.cp in BLACKLIST_PACKAGES:
-        sys.stderr.write(
+        output.ewarn(
             pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp))
         )
-        return []
+        return None
 
     if not CONFIG['quiet']:
+        if not CONFIG['format']:
             pp.uprint(
                 " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
             )
             pp.uprint()
+        else:
+            output.metadata("overlay", pp.section(pkg.repo_name()))
 
         ebuild_path = pkg.ebuild_path()
         if ebuild_path:
-            pp.uprint('Ebuild: ' + pp.path(os.path.normpath(ebuild_path)))
+            output.metadata(
+                "ebuild", pp.path(os.path.normpath(ebuild_path))
+            )
 
-        pp.uprint('Repository: ' + pkg.repo_name())
+        output.metadata("repository", pkg.repo_name())
-        pp.uprint('Homepage: ' + pkg.environment("HOMEPAGE"))
+        output.metadata("homepage", pkg.environment("HOMEPAGE"))
-        pp.uprint('Description: ' + pkg.environment("DESCRIPTION"))
+        output.metadata("description", pkg.environment("DESCRIPTION"))
 
     cpv = pkg.cpv
     metadata = {
@@ -137,15 +157,19 @@ def scan_upstream(query):
         alist = porttree._parse_uri_map(cpv, metadata, use=use)
         aalist = porttree._parse_uri_map(cpv, metadata)
     except Exception as e:
-        sys.stderr.write(pp.warn("%s\n" % str(e)))
+        output.ewarn(pp.warn("%s\n" % str(e)))
-        sys.stderr.write(
+        output.ewarn(
             pp.warn("Invalid SRC_URI for '%s'" % pp.pkgquery(cpv))
         )
-        return []
+        return None
 
     if "mirror" in portage.settings.features:
         urls = aalist
     else:
         urls = alist
 
+    # output scan time for formatted output
+    scan_time = (datetime.now() - start_time).total_seconds()
+    output.metadata("scan_time", scan_time, show=False)
+
     return scan_upstream_urls(pkg.cpv, urls)