euscan: adding json output

Naive json output implmented, probably needs some further tuning

Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
volpino 2012-05-21 22:24:44 +02:00
parent 373fba6e01
commit 8cb19b5a6b
3 changed files with 170 additions and 85 deletions

View File

@ -6,36 +6,43 @@ Distributed under the terms of the GNU General Public License v2
from __future__ import print_function from __future__ import print_function
""" Meta """
# Meta
__author__ = "Corentin Chary (iksaif)" __author__ = "Corentin Chary (iksaif)"
__email__ = "corentin.chary@gmail.com" __email__ = "corentin.chary@gmail.com"
__version__ = "git" __version__ = "git"
__productname__ = "euscan" __productname__ = "euscan"
__description__ = "A tool to detect new upstream releases." __description__ = "A tool to detect new upstream releases."
__version__ = "git"
""" Imports """
# Imports
import sys import sys
import getopt import getopt
import errno import errno
import httplib import httplib
from portage.output import white, yellow, turquoise, green, EOutput from portage.output import white, yellow, turquoise, green
from portage.exception import AmbiguousPackageName from portage.exception import AmbiguousPackageName
from gentoolkit import pprinter as pp from gentoolkit import pprinter as pp
from gentoolkit.eclean.search import (port_settings) from gentoolkit.eclean.search import (port_settings)
from gentoolkit.errors import GentoolkitException from gentoolkit.errors import GentoolkitException
import euscan from euscan import CONFIG, output
from euscan import CONFIG
from euscan.scan import scan_upstream from euscan.scan import scan_upstream
""" Globals """
# Globals
def exit_helper(status):
if CONFIG["format"]:
print(output.get_formatted_output())
sys.exit(status)
def setupSignals(): def setup_signals():
"""This block ensures that ^C interrupts are handled quietly.""" """This block ensures that ^C interrupts are handled quietly."""
import signal import signal
@ -43,14 +50,14 @@ def setupSignals():
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN)
print() print()
sys.exit(errno.EINTR) exit_helper(errno.EINTR)
signal.signal(signal.SIGINT, exithandler) signal.signal(signal.SIGINT, exithandler)
signal.signal(signal.SIGTERM, exithandler) signal.signal(signal.SIGTERM, exithandler)
signal.signal(signal.SIGPIPE, signal.SIG_DFL) signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def printVersion(): def print_version():
"""Output the version info.""" """Output the version info."""
print("%s (%s) - %s" \ print("%s (%s) - %s" \
% (__productname__, __version__, __description__)) % (__productname__, __version__, __description__))
@ -60,23 +67,26 @@ def printVersion():
print("Distributed under the terms of the GNU General Public License v2") print("Distributed under the terms of the GNU General Public License v2")
def printUsage(_error=None, help=None): def print_usage(_error=None, help=None):
"""Print help message. May also print partial help to stderr if an """Print help message. May also print partial help to stderr if an
error from {'options'} is specified.""" error from {'options'} is specified."""
out = sys.stdout out = sys.stdout
if _error: if _error:
out = sys.stderr out = sys.stderr
if not _error in ('global-options', 'packages',): if not _error in ('global-options', 'packages',):
_error = None _error = None
if not _error and not help: if not _error and not help:
help = 'all' help = 'all'
if _error in ('global-options',): if _error in ('global-options',):
print(pp.error("Wrong option on command line."), file=out) output.eerror("Wrong option on command line.\n")
print(file=out)
if _error in ('packages',): if _error in ('packages',):
print(pp.error("You need to specify exactly one package."), file=out) output.eerror("You need to specify exactly one package.\n")
print(file=out)
print(white("Usage:"), file=out) print(white("Usage:"), file=out)
if _error in ('global-options', 'packages',) or help == 'all': if _error in ('global-options', 'packages',) or help == 'all':
print(" " + turquoise(__productname__), print(" " + turquoise(__productname__),
@ -106,14 +116,19 @@ def printUsage(_error=None, help=None):
" (default: 2)\n" + " (default: 2)\n" +
" " * 29 + "bigger levels will generate more versions numbers\n" + " " * 29 + "bigger levels will generate more versions numbers\n" +
" " * 29 + "0 means disabled", file=out) " " * 29 + "0 means disabled", file=out)
print(yellow(" -f, --format=<format>") +
" - define the output " + yellow("<format>") +
" (available: json)", file=out)
print(file=out) print(file=out)
if _error in ('packages',) or help: if _error in ('packages',) or help:
print(green(" package") + print(green(" package") +
" - the packages (or ebuilds) you want to scan", " - the packages (or ebuilds) you want to scan",
file=out) file=out)
print(file=out) print(file=out)
'''print( "More detailed instruction can be found in",
turquoise("`man %s`" % __productname__), file=out)''' #print( "More detailed instruction can be found in",
#turquoise("`man %s`" % __productname__), file=out)
class ParseArgsException(Exception): class ParseArgsException(Exception):
@ -125,12 +140,12 @@ class ParseArgsException(Exception):
return repr(self.value) return repr(self.value)
def parseArgs(): def parse_args():
"""Parse the command line arguments. Raise exceptions on """Parse the command line arguments. Raise exceptions on
errors. Returns package and affect the CONFIG dict. errors. Returns packages and affects the CONFIG dict.
""" """
def optionSwitch(opts): def option_switch(opts):
"""local function for interpreting command line options """local function for interpreting command line options
and setting options accordingly""" and setting options accordingly"""
return_code = True return_code = True
@ -151,29 +166,35 @@ def parseArgs():
CONFIG['brute-force'] = int(a) CONFIG['brute-force'] = int(a)
elif o in ("-v", "--verbose") and not CONFIG['quiet']: elif o in ("-v", "--verbose") and not CONFIG['quiet']:
CONFIG['verbose'] += 1 CONFIG['verbose'] += 1
elif o in ("-f", "--format"):
CONFIG['format'] = a
CONFIG['nocolor'] = True
pp.output.nocolor()
else: else:
return_code = False return_code = False
return return_code return return_code
' here are the different allowed command line options (getopt args) ' # here are the different allowed command line options (getopt args)
getopt_options = {'short': {}, 'long': {}} getopt_options = {'short': {}, 'long': {}}
getopt_options['short']['global'] = "hVCqv1b:" getopt_options['short']['global'] = "hVCqv1bf:"
getopt_options['long']['global'] = ["help", "version", "nocolor", "quiet", getopt_options['long']['global'] = [
"verbose", "oneshot", "brute-force="] "help", "version", "nocolor", "quiet", "verbose", "oneshot",
"brute-force=", "format="
]
short_opts = getopt_options['short']['global'] short_opts = getopt_options['short']['global']
long_opts = getopt_options['long']['global'] long_opts = getopt_options['long']['global']
opts_mode = 'global' opts_mode = 'global'
' apply getopts to command line, show partial help on failure ' # apply getopts to command line, show partial help on failure
try: try:
opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
except: except:
raise ParseArgsException(opts_mode + '-options') raise ParseArgsException(opts_mode + '-options')
' set options accordingly ' # set options accordingly
optionSwitch(opts) option_switch(opts)
if len(args) < 1: if len(args) < 1:
raise ParseArgsException('packages') raise ParseArgsException('packages')
@ -183,29 +204,32 @@ def parseArgs():
def main(): def main():
"""Parse command line and execute all actions.""" """Parse command line and execute all actions."""
CONFIG['nocolor'] = (port_settings["NOCOLOR"] in ('yes', 'true') CONFIG['nocolor'] = (
or not sys.stdout.isatty()) port_settings["NOCOLOR"] in ('yes', 'true') or not sys.stdout.isatty()
)
if CONFIG['nocolor']: if CONFIG['nocolor']:
pp.output.nocolor() pp.output.nocolor()
' parse command line options and actions '
# parse command line options and actions
try: try:
packages = parseArgs() packages = parse_args()
except ParseArgsException as e: except ParseArgsException as e:
if e.value == 'help': if e.value == 'help':
printUsage(help='all') print_usage(help='all')
sys.exit(0) exit_helper(0)
elif e.value[:5] == 'help-':
printUsage(help=e.value[5:]) elif e.value[:5] == 'help-':
sys.exit(0) print_usage(help=e.value[5:])
elif e.value == 'version': exit_helper(0)
printVersion()
sys.exit(0) elif e.value == 'version':
else: print_version()
printUsage(e.value) exit_helper(0)
sys.exit(errno.EINVAL)
else:
print_usage(e.value)
exit_helper(errno.EINVAL)
""" Change euscan's output """
euscan.output = EOutput(CONFIG['quiet'])
if CONFIG['verbose'] > 2: if CONFIG['verbose'] > 2:
httplib.HTTPConnection.debuglevel = 1 httplib.HTTPConnection.debuglevel = 1
@ -216,48 +240,39 @@ def main():
ret = scan_upstream(package) ret = scan_upstream(package)
except AmbiguousPackageName as e: except AmbiguousPackageName as e:
pkgs = e.args[0] pkgs = e.args[0]
for candidate in pkgs: output.eerror("\n".join(pkgs))
print(candidate)
from os.path import basename # To get the short name from os.path import basename # To get the short name
print(file=sys.stderr) output.error(
print( "The short ebuild name '%s' is ambiguous. Please specify" %
pp.error( basename(pkgs[0]),
"The short ebuild name '%s' is ambiguous. Please specify" \ "one of the above fully-qualified ebuild names instead."
% basename(pkgs[0])
),
file=sys.stderr, end=""
) )
pp.die(1, "one of the above fully-qualified ebuild names instead.") exit_helper(1)
except GentoolkitException as err:
pp.die(1, '%s: %s' % (package, str(err)))
except Exception as err:
pp.die(1, '%s: %s' % (package, str(err)))
if not CONFIG['quiet']: except GentoolkitException as err:
output.eerror('%s: %s' % (package, str(err)))
exit_helper(1)
except Exception as err:
output.eerror('%s: %s' % (package, str(err)))
exit_helper(1)
if not CONFIG['quiet'] and not CONFIG['format']:
print() print()
for cp, url, version in ret: for cp, url, version in ret:
if not CONFIG['quiet']: output.result(cp, version, url)
print("Upstream Version: "
+ pp.number("%s" % version)
+ pp.path(" %s" % url))
else:
print(pp.cpv("%s-%s" % (cp, version))
+ ": " + pp.path(url))
if not len(ret) and not CONFIG['quiet']: if not len(ret) and not CONFIG['quiet']:
print(pp.warn("Didn't find any new version, " output.ewarn(
+ "check package's homepage for " "Didn't find any new version, check package's homepage for " +
+ "more informations")) "more informations"
)
if __name__ == "__main__": if __name__ == "__main__":
try: setup_signals()
setupSignals()
main() main()
except KeyboardInterrupt: exit_helper(0)
print("Aborted.")
sys.exit(errno.EINTR)
sys.exit(0)

View File

@ -3,11 +3,17 @@
# Copyright 2011 Corentin Chary <corentin.chary@gmail.com> # Copyright 2011 Corentin Chary <corentin.chary@gmail.com>
# Distributed under the terms of the GNU General Public License v2 # Distributed under the terms of the GNU General Public License v2
from io import StringIO
from collections import defaultdict
import json
from gentoolkit import pprinter as pp
from portage.output import EOutput
__version__ = "git" __version__ = "git"
from portage.output import EOutput
CONFIG = { CONFIG = {
'nocolor': False, 'nocolor': False,
'quiet': False, 'quiet': False,
@ -20,11 +26,11 @@ CONFIG = {
'oneshot': True, 'oneshot': True,
'user-agent': 'escan (http://euscan.iksaif.net)', 'user-agent': 'escan (http://euscan.iksaif.net)',
'skip-robots-txt': False, 'skip-robots-txt': False,
'cache': False 'cache': False,
'format': None,
'indent': 2
} }
output = EOutput(CONFIG['quiet'])
BLACKLIST_VERSIONS = [ BLACKLIST_VERSIONS = [
# Compatibility package for running binaries linked against a # Compatibility package for running binaries linked against a
# pre gcc 3.4 libstdc++, won't be updated # pre gcc 3.4 libstdc++, won't be updated
@ -67,3 +73,67 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
'(.*)chromium.org(.*)', '(.*)chromium.org(.*)',
'(.*)nodejs.org(.*)', '(.*)nodejs.org(.*)',
] ]
class EOutputFile(EOutput):
"""
Override of EOutput, allows to specify an output file for writes
"""
def __init__(self, out_file=None, *args, **kwargs):
super(EOutputFile, self).__init__(*args, **kwargs)
self.out_file = out_file
def _write(self, f, msg):
if self.out_file is None:
super(EOutputFile, self)._write(f, msg)
else:
super(EOutputFile, self)._write(self.out_file, msg)
class EuscanOutput(object):
"""
Class that handles output for euscan
"""
def __init__(self, config):
self.config = config
self.data = defaultdict(StringIO)
self.packages = defaultdict(list)
def get_formatted_output(self):
data = {}
for key in self.data:
if key not in ("ebegin", "eend"):
val = [x for x in self.data[key].getvalue().split("\n") if x]
data[key] = val
data["result"] = self.packages
if self.config["format"].lower() == "json":
return json.dumps(data, indent=self.config["indent"])
else:
raise TypeError("Invalid output format")
def result(self, cp, version, url):
if self.config['format']:
self.packages[cp].append({"version": version, "url": url})
else:
if not self.config['quiet']:
print "Upstream Version:", pp.number("%s" % version),
print pp.path(" %s" % url)
else:
print pp.cpv("%s-%s" % (cp, version)) + ":", pp.path(url)
def __getattr__(self, key):
output_file = self.data[key] if self.config["format"] else None
if output_file:
_output = EOutputFile(out_file=self.data[key],
quiet=self.config['quiet'])
ret = getattr(_output, key)
else:
ret = getattr(EOutputFile(quiet=self.config['quiet']), key)
return ret
output = EuscanOutput(CONFIG)

View File

@ -39,7 +39,7 @@ def scan_upstream_urls(cpv, urls):
for filename in urls: for filename in urls:
for url in urls[filename]: for url in urls[filename]:
if not CONFIG['quiet']: if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint() pp.uprint()
euscan.output.einfo("SRC_URI is '%s'" % url) euscan.output.einfo("SRC_URI is '%s'" % url)
@ -101,7 +101,7 @@ def scan_upstream(query):
) )
return [] return []
if not CONFIG['quiet']: if not CONFIG['quiet'] and not CONFIG['format']:
pp.uprint( pp.uprint(
" * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name())) " * %s [%s]" % (pp.cpv(pkg.cpv), pp.section(pkg.repo_name()))
) )